diff --git a/CHANGELOG.md b/CHANGELOG.md index a7fc880408..ace06dcd22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Server-side identity providers for ID5 (`[integrations.id5]`), LiveIntent (`[integrations.liveintent]`), LiveRamp ATS (`[integrations.liveramp]`), UID2 (`[integrations.uid2]`) and Yahoo ConnectID (`[integrations.connectid]`), and revoke-only Lockr capture (`[integrations.lockr].capture_identity`). Each source they write needs an `[[ec.partners]]` entry. See [Identity Vendors](docs/guide/integrations/identity-vendors.md). +- `tsjs.setEmail`, `tsjs.setHashedEmail` and `tsjs.clearHashedEmail` store the SHA-256 of the visitor's normalized email in the first-party `ts-hem` cookie, the identity providers' hashed-email input. Trusted Server reads it per request, never stores it, and strips it from requests to the origin and to Prebid Server. +- Identity providers can declare and read first-party cookies with `Resolves::with_cookie` and `ResolveInput::cookie`. - Server-side identity provider framework. An integration registers an `IdentityProvider` with `try_with_identity_provider`, and the coordinator handles consent, transport limits, backoff and the single KV write. Provider authors get a test harness (`ProviderHarness`, `check_conformance`, `output_eids`). See [Writing an Identity Provider](docs/guide/identity-providers.md). No vendor ships with the framework. - Source ownership: `owner` on an `[[ec.partners]]` entry names the provider that owns a source. An owner's result replaces the source's whole entry set, and batch sync writes an owned source only when the row holds no value for it (an expired owned record counts as present and is not filled). Validation rejects an owner that is not enabled, is capture-only, does not claim the source, or collides with another provider. See [Source ownership](docs/guide/configuration.md#source-ownership-ec-partners-owner). - Hold mode: `mode = "hold"` in `[ec.identity.providers.]` runs a provider before the navigation's auction, within the shared `hold_budget_ms`, with `timeout_ms` and `after_send_timeout_ms` as call timeouts. diff --git a/Cargo.lock b/Cargo.lock index 00dff9fc05..1fc4ab6ed5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,31 @@ dependencies = [ "generic-array", ] +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -1114,6 +1139,15 @@ dependencies = [ "linktime-proc-macro", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -2181,6 +2215,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gix" version = "0.83.0" @@ -4600,6 +4644,18 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -4761,7 +4817,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -4781,7 +4837,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.118", @@ -4794,7 +4850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.118", @@ -6636,6 +6692,7 @@ dependencies = [ name = "trusted-server-core" version = "0.1.0" dependencies = [ + "aes-gcm", "async-stream", "async-trait", "base64", diff --git a/Cargo.toml b/Cargo.toml index c0d79bc033..9be2f274d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ assert_cmd = "2" async-stream = "0.3" async-trait = "0.1" axum = "0.8" +aes-gcm = "0.10" base64 = "0.22" brotli = "8.0" build-print = "1.0.1" diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index 0a8d052805..98a7031533 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -11,6 +11,7 @@ version = { workspace = true } workspace = true [dependencies] +aes-gcm = { workspace = true } async-trait = { workspace = true } async-stream = { workspace = true } base64 = { workspace = true } diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 28a6de49f4..8eea6f5fc2 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -1148,6 +1148,8 @@ mod tests { user_agent: None, dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }, ); let cookie = cookie_only_ts_eids(); diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 6b9d86cd73..87e6387517 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -21,19 +21,24 @@ use crate::error::TrustedServerError; use crate::integrations::{ adserver_mock::AdServerMockConfig, aps::ApsConfig, + connectid::{CONNECTID_INTEGRATION_ID, ConnectIdConfig}, datadome::DataDomeConfig, didomi::DidomiIntegrationConfig, google_tag_manager::GoogleTagManagerConfig, gpt::GptConfig, gpt_diagnostics::GptDiagnosticsConfig, + id5::{ID5_INTEGRATION_ID, Id5Config}, js_asset_proxy::{JS_ASSET_PROXY_INTEGRATION_ID, JsAssetProxyConfig}, - lockr::LockrConfig, + liveintent::{LIVEINTENT_INTEGRATION_ID, LiveIntentConfig}, + liveramp::{LIVERAMP_INTEGRATION_ID, LiveRampConfig}, + lockr::{LOCKR_INTEGRATION_ID, LockrConfig}, nextjs::NextJsIntegrationConfig, osano::OsanoConfig, permutive::PermutiveConfig, prebid, sourcepoint::SourcepointConfig, testlight::TestlightConfig, + uid2::{UID2_INTEGRATION_ID, Uid2Config}, }; use crate::settings::{ AssetOriginAuth, IdentitySettings, IntegrationConfig, ProviderMode, Settings, @@ -62,6 +67,11 @@ const DEPLOY_VALIDATED_INTEGRATION_IDS: &[&str] = &[ "datadome", "gpt", "gpt_diagnostics", + ID5_INTEGRATION_ID, + LIVEINTENT_INTEGRATION_ID, + LIVERAMP_INTEGRATION_ID, + UID2_INTEGRATION_ID, + CONNECTID_INTEGRATION_ID, JS_ASSET_PROXY_INTEGRATION_ID, ]; @@ -203,6 +213,22 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { ], true, ), + field( + vec![ + optional_object("integrations"), + optional_object("uid2"), + object("api_key"), + ], + true, + ), + field( + vec![ + optional_object("integrations"), + optional_object("uid2"), + object("client_secret"), + ], + true, + ), field( vec![ optional_object("integrations"), @@ -322,6 +348,37 @@ pub(crate) fn identity_descriptors_for_deploy( #[allow(unused_mut)] let mut descriptors = crate::ec::identity::generic_pull::generic_pull_descriptors(&settings.ec.partners)?; + descriptors.extend(validate_identity_integration::( + settings, + ID5_INTEGRATION_ID, + )?); + descriptors.extend(validate_identity_integration::( + settings, + LIVEINTENT_INTEGRATION_ID, + )?); + descriptors.extend(validate_identity_integration::( + settings, + LIVERAMP_INTEGRATION_ID, + )?); + descriptors.extend(validate_identity_integration::( + settings, + UID2_INTEGRATION_ID, + )?); + descriptors.extend(validate_identity_integration::( + settings, + CONNECTID_INTEGRATION_ID, + )?); + if let Some(lockr) = settings.integration_config::(LOCKR_INTEGRATION_ID)? + && lockr.capture_identity + { + descriptors.push(lockr.build_descriptor().change_context( + TrustedServerError::Configuration { + message: format!( + "integrations.{LOCKR_INTEGRATION_ID}: the identity provider cannot be described" + ), + }, + )?); + } #[cfg(test)] descriptors.extend(validate_identity_integration::< crate::ec::identity::examples::ExampleIdConfig, @@ -343,8 +400,6 @@ pub(crate) fn identity_descriptors_for_deploy( /// Returns [`TrustedServerError::Configuration`] when the integration's /// config does not parse or validate, or its descriptor cannot be built. /// Returns `Ok(None)` when the integration is absent or disabled. -// Temporary: the first production identity integration adds the first caller. -#[allow(dead_code)] pub(crate) fn validate_identity_integration( settings: &Settings, id: &str, @@ -477,6 +532,11 @@ fn validate_enabled_integrations( validate_integration::(settings, "nextjs")?; validate_integration::(settings, "permutive")?; validate_integration::(settings, "lockr")?; + validate_integration::(settings, ID5_INTEGRATION_ID)?; + validate_integration::(settings, LIVEINTENT_INTEGRATION_ID)?; + validate_integration::(settings, LIVERAMP_INTEGRATION_ID)?; + validate_integration::(settings, UID2_INTEGRATION_ID)?; + validate_integration::(settings, CONNECTID_INTEGRATION_ID)?; validate_integration::(settings, "didomi")?; validate_integration::(settings, "sourcepoint")?; validate_integration::(settings, "osano")?; @@ -1084,6 +1144,8 @@ formats = [{ width = 300, height = 250 }] "integrations.datadome.server_side_key_secret_name".to_owned(), true, ), + ("integrations.uid2.api_key".to_owned(), true), + ("integrations.uid2.client_secret".to_owned(), true), ( "integrations.datadome.protection_test_bypass.credential_secret_name" .to_owned(), diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index da9ea740e8..77290522a4 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -10,6 +10,10 @@ pub const COOKIE_TS_EIDS: &str = "ts-eids"; /// Client-only mirror of the servable server-side EIDs, written by the shim /// and never read by the server (spec 6.2). pub const COOKIE_TS_EIDS_S: &str = "__Host-ts-eids-s"; +/// Cookie written by the Trusted Server JS SDK holding the lowercase hex +/// SHA-256 of the visitor's normalized email address. Read per request as the +/// identity providers' hashed-email input; never stored in KV. +pub const COOKIE_TS_HEM: &str = "ts-hem"; pub const COOKIE_TS_TESTER: &str = "ts-tester"; pub const COOKIE_SHAREDID: &str = "sharedId"; diff --git a/crates/trusted-server-core/src/cookies.rs b/crates/trusted-server-core/src/cookies.rs index 43105eb6ae..2b0f3227da 100644 --- a/crates/trusted-server-core/src/cookies.rs +++ b/crates/trusted-server-core/src/cookies.rs @@ -13,7 +13,7 @@ use http::header; use crate::constants::{ COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_TS_EC, COOKIE_TS_EIDS, - COOKIE_TS_EIDS_S, COOKIE_US_PRIVACY, + COOKIE_TS_EIDS_S, COOKIE_TS_HEM, COOKIE_US_PRIVACY, }; use crate::error::TrustedServerError; @@ -33,18 +33,26 @@ pub const CONSENT_COOKIE_NAMES: &[&str] = &[ /// /// Stripped from Prebid Server requests and never added to proxy or provider /// requests. -pub const TS_IDENTITY_COOKIE_NAMES: &[&str] = &[COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_TS_EIDS_S]; +pub const TS_IDENTITY_COOKIE_NAMES: &[&str] = &[ + COOKIE_TS_EC, + COOKIE_TS_EIDS, + COOKIE_TS_EIDS_S, + COOKIE_TS_HEM, +]; /// Identity cookies removed from publisher-origin requests by default. /// /// `ts-ec` and `ts-eids` still reach the origin, as they did before the /// `ts-eids-s` mirror existed, so the origin's `Cookie` header stays at its -/// earlier size. See [`TS_ORIGIN_STRIPPED_COOKIE_NAMES_WITH_EIDS`]. -pub const TS_ORIGIN_STRIPPED_COOKIE_NAMES: &[&str] = &[COOKIE_TS_EIDS_S]; +/// earlier size. The hashed email never leaves the edge. See +/// [`TS_ORIGIN_STRIPPED_COOKIE_NAMES_WITH_EIDS`]. +pub const TS_ORIGIN_STRIPPED_COOKIE_NAMES: &[&str] = &[COOKIE_TS_EIDS_S, COOKIE_TS_HEM]; /// Identity cookies removed from publisher-origin requests when the operator -/// sets `ec.strip_eids_cookie_from_origin`: `ts-eids` as well as the mirror. -pub const TS_ORIGIN_STRIPPED_COOKIE_NAMES_WITH_EIDS: &[&str] = &[COOKIE_TS_EIDS, COOKIE_TS_EIDS_S]; +/// sets `ec.strip_eids_cookie_from_origin`: `ts-eids` as well as the mirror +/// and the hashed email. +pub const TS_ORIGIN_STRIPPED_COOKIE_NAMES_WITH_EIDS: &[&str] = + &[COOKIE_TS_EIDS, COOKIE_TS_EIDS_S, COOKIE_TS_HEM]; /// Parses a cookie string into a [`CookieJar`]. /// @@ -520,14 +528,15 @@ mod tests { #[test] fn strip_cookie_pairs_removes_exact_names_only() { - let header = - "ts-ec=abc; ts-eids=W10; __Host-ts-eids-s=eyJ2IjoxfQ; other=1; ts-eids-extra=keep"; + let header = "ts-ec=abc; ts-eids=W10; __Host-ts-eids-s=eyJ2IjoxfQ; ts-hem=00ff; other=1; \ + ts-eids-extra=keep"; let stripped = strip_cookie_pairs(header, TS_ORIGIN_STRIPPED_COOKIE_NAMES_WITH_EIDS); assert_eq!( stripped, "ts-ec=abc; other=1; ts-eids-extra=keep", - "should drop the ID cookies and keep ts-ec and a cookie that only shares a prefix" + "should drop the ID and hashed-email cookies and keep ts-ec and a cookie that only \ + shares a prefix" ); assert_eq!( diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 0508b5a2b1..a93da3dcac 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1939,6 +1939,8 @@ mod tests { user_agent: None, dnt: false, page_url: None, + hashed_email: None, + cookies: std::collections::BTreeMap::new(), }); } diff --git a/crates/trusted-server-core/src/ec/identity/coordinator.rs b/crates/trusted-server-core/src/ec/identity/coordinator.rs index 46bbb23a6e..651be03780 100644 --- a/crates/trusted-server-core/src/ec/identity/coordinator.rs +++ b/crates/trusted-server-core/src/ec/identity/coordinator.rs @@ -4495,6 +4495,7 @@ mod tests { random: &ThreadRandom, ec_id: Some(format!("{}.ABC123", "a".repeat(64))), stored: BTreeMap::new(), + cookies: BTreeMap::new(), }; futures::executor::block_on(dispatch(&deps, calls, &signals, budget)) } @@ -5869,6 +5870,8 @@ mod tests { user_agent: Some("ExampleBrowser/1.0".to_owned()), dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }); let mut writer = HoldFixture::new( vec![hold_provider("p", HOLD_HOST, HOLD_SOURCE)], @@ -5935,6 +5938,8 @@ mod tests { user_agent: Some("ExampleBrowser/1.0".to_owned()), dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }); fixture.run_hold(); } @@ -5948,6 +5953,8 @@ mod tests { user_agent: Some("ExampleBrowser/1.0".to_owned()), dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }); fixture.run_hold(); @@ -6229,6 +6236,8 @@ mod tests { user_agent: Some("ExampleBrowser/1.0".to_owned()), dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }); fixture.capture( @@ -6730,6 +6739,8 @@ mod tests { user_agent: None, dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }, ); diff --git a/crates/trusted-server-core/src/ec/identity/descriptor.rs b/crates/trusted-server-core/src/ec/identity/descriptor.rs index 908ea9df39..e595016704 100644 --- a/crates/trusted-server-core/src/ec/identity/descriptor.rs +++ b/crates/trusted-server-core/src/ec/identity/descriptor.rs @@ -9,6 +9,7 @@ use url::Url; use super::ids::{IdentityProviderId, SourceDomain}; use super::resolution::IdentityProviderError; +use crate::cookies::{CONSENT_COOKIE_NAMES, TS_IDENTITY_COOKIE_NAMES}; use crate::ec::kv_types::{MAX_EID_FIELD_BYTES, is_valid_eid_field}; use crate::ec::partner::normalize_partner_source_domain; @@ -253,6 +254,16 @@ impl ProviderDescriptorBuilder { ))); } } + for name in &resolves.cookies { + if !is_cookie_name(name) { + return Err(fail(&format!("cookie `{name}` is not a valid cookie name"))); + } + if is_reserved_cookie(name) { + return Err(fail(&format!( + "cookie `{name}` is a Trusted Server identity or consent cookie" + ))); + } + } if let Some(gap) = self.consent_policy.resolver_gap() { return Err(fail(gap)); } @@ -277,6 +288,23 @@ impl ProviderDescriptorBuilder { } } +/// Whether `name` is an RFC 6265 cookie-name token. +fn is_cookie_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|byte| byte.is_ascii_graphic() && !b"()<>@,;:\\\"/[]?={}".contains(&byte)) +} + +/// Trusted Server's own identity cookies and the consent cookies, which no +/// provider reads directly. +fn is_reserved_cookie(name: &str) -> bool { + TS_IDENTITY_COOKIE_NAMES + .iter() + .chain(CONSENT_COOKIE_NAMES) + .any(|reserved| reserved.eq_ignore_ascii_case(name)) +} + /// The kind of resolution one source needs. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display)] #[non_exhaustive] @@ -301,6 +329,7 @@ pub struct Resolves { private_state: bool, refresh_grace: Option, ec_id: bool, + cookies: BTreeSet, } impl Default for Resolves { @@ -323,6 +352,7 @@ impl Resolves { private_state: false, refresh_grace: None, ec_id: false, + cookies: BTreeSet::new(), } } @@ -396,6 +426,17 @@ impl Resolves { self } + /// A first-party cookie the provider may read, such as the vendor's own + /// ID cookie on the publisher's domain. Repeatable. It is passed when the + /// request carries it and is never required. `build()` refuses Trusted + /// Server's identity cookies and the consent cookies, which providers + /// read through [`ResolveInput::consent`](super::input::ResolveInput::consent). + #[must_use] + pub fn with_cookie(mut self, name: &str) -> Self { + self.cookies.insert(name.to_owned()); + self + } + /// Gives the provider the full EC ID. Only generic pull may use this. #[must_use] pub(in crate::ec::identity) fn with_ec_id(mut self) -> Self { @@ -460,6 +501,12 @@ impl Resolves { pub(crate) fn gets_ec_id(&self) -> bool { self.ec_id } + + /// The first-party cookies the provider may read. + #[must_use] + pub(crate) fn cookies(&self) -> &BTreeSet { + &self.cookies + } } /// Revoke-only capture from responses TS proxies for this provider: it @@ -765,7 +812,8 @@ pub enum Input { /// Country and subdivision only. #[display("region")] Region, - /// A SHA-256 of a normalized email address. Never available in v1. + /// A SHA-256 of a normalized email address, from the `ts-hem` cookie the + /// JS SDK writes. Never stored. #[display("hashed_email")] HashedEmail, } @@ -786,6 +834,9 @@ pub enum Access { /// The full EC ID, without `with_ec_id()`. Only generic pull declares it. #[display("ec_id")] EcId, + /// A first-party cookie, without `with_cookie()` for its name. + #[display("cookie")] + Cookie, } #[cfg(test)] @@ -930,6 +981,30 @@ mod tests { ); } + #[test] + fn keeps_declared_cookies() { + let descriptor = ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie("_vendor_fpid") + .with_cookie("_vendor_fpid"), + ) + .with_consent_policy(crate::ec::identity::test_support::resolver_policy()) + .build() + .expect("should build"); + + let cookies: Vec<&str> = descriptor + .resolves() + .expect("should resolve") + .cookies() + .iter() + .map(String::as_str) + .collect(); + assert_eq!(cookies, ["_vendor_fpid"], "should keep each name once"); + } + #[test] fn build_rejects_invalid_descriptors() { let cases: Vec<(&str, ProviderDescriptorBuilder)> = vec![ @@ -1035,6 +1110,46 @@ mod tests { .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) .with_provenance(&source("example.com"), EidProvenance::new().with_mm(-1)), ), + ( + "cookie name with a separator", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie("vendor;id"), + ), + ), + ( + "empty cookie name", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie(""), + ), + ), + ( + "Trusted Server identity cookie", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie("TS-HEM"), + ), + ), + ( + "consent cookie", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie("euconsent-v2"), + ), + ), ]; for (name, builder) in cases { let error = builder diff --git a/crates/trusted-server-core/src/ec/identity/flush.rs b/crates/trusted-server-core/src/ec/identity/flush.rs index c2969b4fb8..c8383aa38a 100644 --- a/crates/trusted-server-core/src/ec/identity/flush.rs +++ b/crates/trusted-server-core/src/ec/identity/flush.rs @@ -457,6 +457,7 @@ pub(super) fn request_facts(context: &EcContext, now: u64, runs_auction: bool) - (Input::UserAgent, identity.user_agent().is_some()), (Input::PageUrl, identity.page_url().is_some()), (Input::Region, region(context).is_some()), + (Input::HashedEmail, identity.hashed_email().is_some()), ] { if present { available_inputs.insert(input); @@ -608,13 +609,14 @@ pub(super) async fn add_dispatch_results( user_agent: identity.user_agent(), page_url: identity.page_url().cloned(), region: region(input.context), - hashed_email: None, + hashed_email: identity.hashed_email().cloned(), consent: ProviderConsent::from_context(input.context.consent()), publisher_domain: &deps.settings.publisher.domain, now: input.facts.now, random: deps.random, ec_id: Some(input.ec_id.to_owned()), stored, + cookies: identity.cookies().clone(), }; let catalog = deps.catalog; for (id, outcome) in dispatch(deps, calls, &signals, budget).await { @@ -803,6 +805,8 @@ mod tests { user_agent: Some("ExampleBrowser/1.0".to_owned()), dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }); if let Some(row) = row { graph.create(TEST_EC_ID, row).expect("should seed the row"); @@ -1509,6 +1513,8 @@ mod tests { user_agent: None, dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }); assert_eq!( diff --git a/crates/trusted-server-core/src/ec/identity/input.rs b/crates/trusted-server-core/src/ec/identity/input.rs index ef64652df4..5446b4d425 100644 --- a/crates/trusted-server-core/src/ec/identity/input.rs +++ b/crates/trusted-server-core/src/ec/identity/input.rs @@ -162,8 +162,22 @@ impl fmt::Debug for HashedEmail { } impl HashedEmail { - /// From digest bytes. Test support only: v1 has no source of hashed - /// emails, so nothing else builds one. + /// From the `ts-hem` cookie value: 64 hex digits of the digest, in + /// either case. `None` for anything else, so a malformed cookie is + /// treated as absent. + #[must_use] + pub(crate) fn from_cookie_hex(value: &str) -> Option { + let value = value.trim(); + if value.len() != 64 { + return None; + } + let mut bytes = [0u8; 32]; + hex::decode_to_slice(value, &mut bytes).ok()?; + Some(Self(bytes)) + } + + /// From digest bytes. Test support only: the one runtime source is the + /// `ts-hem` cookie, read with [`Self::from_cookie_hex`]. #[cfg(any(test, feature = "test-utils"))] #[must_use] pub fn from_bytes(bytes: [u8; 32]) -> Self { @@ -432,6 +446,9 @@ pub(crate) struct ResolveSignals<'a> { pub(crate) random: &'a dyn RandomSource, pub(crate) ec_id: Option, pub(crate) stored: BTreeMap, + /// The request's first-party cookies other than Trusted Server's own and + /// the consent cookies. A provider reads only the names it declared. + pub(crate) cookies: BTreeMap, } /// What one provider may read for one call. @@ -675,7 +692,7 @@ impl<'a> ResolveInput<'a> { self.declared(Input::Region, self.signals.region.as_ref()) } - /// A hashed email. Never available in v1. + /// The hashed email from the `ts-hem` cookie. /// /// # Errors /// @@ -684,6 +701,25 @@ impl<'a> ResolveInput<'a> { self.declared(Input::HashedEmail, self.signals.hashed_email.as_ref()) } + /// The value of the first-party cookie `name`, or `None` when the request + /// does not carry it. + /// + /// # Errors + /// + /// `UndeclaredInput { access: Cookie }` when the descriptor did not + /// declare `name` with + /// [`Resolves::with_cookie`](super::descriptor::Resolves::with_cookie). + pub fn cookie(&self, name: &str) -> Result, Report> { + if !self + .descriptor + .resolves() + .is_some_and(|resolves| resolves.cookies().contains(name)) + { + return Err(self.record(Access::Cookie)); + } + Ok(self.signals.cookies.get(name).map(String::as_str)) + } + /// The full EC ID. Only for a resolver declared `with_ec_id()`. /// /// # Errors @@ -827,9 +863,58 @@ mod tests { random: &ThreadRandom, ec_id: Some("a".repeat(64) + ".ABC123"), stored: BTreeMap::new(), + cookies: BTreeMap::from([("_vendor_fpid".to_owned(), "fpid-value".to_owned())]), } } + #[test] + fn declared_cookies_are_readable_and_undeclared_ones_are_recorded() { + let descriptor = descriptor( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie("_vendor_fpid") + .with_cookie("_vendor_other"), + ); + let signals = signals(); + let input = ResolveInput::new( + &descriptor, + kinds(&descriptor, ResolveKind::Initial), + &signals, + ); + + assert_eq!( + input + .cookie("_vendor_fpid") + .expect("should read a declared cookie"), + Some("fpid-value"), + "should return the cookie value" + ); + assert_eq!( + input + .cookie("_vendor_other") + .expect("should read a declared cookie"), + None, + "should return None for a declared cookie the request lacks" + ); + let error = input + .cookie("_undeclared") + .expect_err("should refuse an undeclared cookie"); + assert!( + matches!( + error.current_context(), + IdentityProviderError::UndeclaredInput { + access: Access::Cookie + } + ), + "should report UndeclaredInput" + ); + assert_eq!( + input.recorded_errors(), + [Access::Cookie], + "should record the undeclared read" + ); + } + #[test] fn declared_inputs_are_available_and_wanted_missing_ones_are_unavailable() { let descriptor = descriptor( @@ -1195,6 +1280,31 @@ mod tests { ); } + #[test] + fn hashed_email_parses_the_cookie_hex_in_either_case() { + let lower = HashedEmail::from_cookie_hex(&"ab".repeat(32)).expect("should parse lowercase"); + let upper = HashedEmail::from_cookie_hex(&"AB".repeat(32)).expect("should parse uppercase"); + + assert_eq!(lower, upper, "should read both cases as one digest"); + assert_eq!(lower.as_bytes(), &[0xab; 32], "should decode the digest"); + } + + #[test] + fn hashed_email_rejects_a_cookie_that_is_not_a_digest() { + for value in [ + "", + "ab", + &"ab".repeat(31), + &"ab".repeat(33), + &"zz".repeat(32), + ] { + assert!( + HashedEmail::from_cookie_hex(value).is_none(), + "should reject `{value}`" + ); + } + } + #[test] fn capture_input_exposes_path_and_never_the_request_body() { let descriptor = diff --git a/crates/trusted-server-core/src/ec/identity/request.rs b/crates/trusted-server-core/src/ec/identity/request.rs index 2ff20fb30b..81b52076c8 100644 --- a/crates/trusted-server-core/src/ec/identity/request.rs +++ b/crates/trusted-server-core/src/ec/identity/request.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use url::Url; use super::ids::{IdentityProviderId, SourceDomain}; +use super::input::HashedEmail; use super::mutation::{IdentityMutation, is_opted_out}; use super::runtime::IdentityRuntime; use crate::ec::kv_types::KvEntry; @@ -38,6 +39,8 @@ pub(crate) struct RequestIdentity { user_agent: Option, dnt: bool, page_url: Option, + hashed_email: Option, + cookies: BTreeMap, runtime: Option>, rate_limiter: Option>, runs_auction: bool, @@ -104,6 +107,19 @@ impl RequestIdentity { self.page_url.as_ref() } + /// The hashed email from the request's `ts-hem` cookie. + #[must_use] + pub(crate) fn hashed_email(&self) -> Option<&HashedEmail> { + self.hashed_email.as_ref() + } + + /// The request's first-party cookies other than Trusted Server's own and + /// the consent cookies, for providers that declare them. + #[must_use] + pub(crate) fn cookies(&self) -> &BTreeMap { + &self.cookies + } + /// The request's identity runtime, when the adapter set one. #[must_use] pub(crate) fn runtime(&self) -> Option<&Arc> { @@ -279,6 +295,8 @@ impl RequestIdentity { self.user_agent = decision.user_agent; self.dnt = decision.dnt; self.page_url = decision.page_url; + self.hashed_email = decision.hashed_email; + self.cookies = decision.cookies; } /// Whether the writer is a `/_ts/page-bids` call, which the interval @@ -312,6 +330,8 @@ pub(crate) struct WriterDecision { pub(crate) user_agent: Option, pub(crate) dnt: bool, pub(crate) page_url: Option, + pub(crate) hashed_email: Option, + pub(crate) cookies: BTreeMap, } #[cfg(test)] diff --git a/crates/trusted-server-core/src/ec/identity/test_support.rs b/crates/trusted-server-core/src/ec/identity/test_support.rs index 4ada943441..f82751990d 100644 --- a/crates/trusted-server-core/src/ec/identity/test_support.rs +++ b/crates/trusted-server-core/src/ec/identity/test_support.rs @@ -901,6 +901,7 @@ pub struct HarnessSetup { state_json: Option, now: Option, random_seed: Option, + cookies: BTreeMap, } impl HarnessSetup { @@ -956,6 +957,13 @@ impl HarnessSetup { self } + /// Adds a first-party cookie the request carries. Repeatable. + #[must_use] + pub fn with_cookie(mut self, name: &str, value: &str) -> Self { + self.cookies.insert(name.to_owned(), value.to_owned()); + self + } + /// Sets the raw consent strings. Default: GDPR does not apply and there /// are no opt-outs. #[must_use] @@ -1361,6 +1369,12 @@ impl<'provider> ProviderHarness<'provider> { self.map_setup(|setup| setup.with_hashed_email(email)) } + /// Adds a first-party cookie the request carries. Repeatable. + #[must_use] + pub fn with_cookie(self, name: &str, value: &str) -> Self { + self.map_setup(|setup| setup.with_cookie(name, value)) + } + /// Sets the raw consent strings. Default: GDPR does not apply and no /// opt-outs. #[must_use] @@ -1896,6 +1910,7 @@ impl<'provider> ProviderHarness<'provider> { random: &self.random, ec_id: self.ec_id.clone(), stored: self.stored_by_source(), + cookies: self.setup.cookies.clone(), } } } @@ -2773,6 +2788,8 @@ pub(crate) mod hold_fixture { user_agent: Some("ExampleBrowser/1.0".to_owned()), dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }); context.identity_mut().set_runs_auction(runs_auction); context.set_kv_snapshot(graph.load_snapshot(TEST_EC_ID)); @@ -2809,6 +2826,8 @@ pub(crate) mod hold_fixture { user_agent: Some("ExampleBrowser/1.0".to_owned()), dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }); context.identity_mut().set_runs_auction(true); context @@ -2955,6 +2974,8 @@ pub(crate) mod capture_fixture { user_agent: Some("ExampleBrowser/1.0".to_owned()), dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }); context.set_kv_snapshot(graph.load_snapshot(TEST_EC_ID)); store.reset_counts(); diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 6b1d4706f7..b87ccb6a83 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -68,6 +68,7 @@ pub fn log_id(ec_id: &str) -> String { format!("{prefix}\u{2026}") } +use std::collections::BTreeMap; use std::sync::Arc; use cookie::CookieJar; @@ -77,8 +78,8 @@ use http::{Request, header}; use url::Url; use crate::consent::{self as consent_mod, ConsentContext, ConsentPipelineInput}; -use crate::constants::{COOKIE_TS_EC, COOKIE_TS_EC_REVOKED}; -use crate::cookies::handle_request_cookies; +use crate::constants::{COOKIE_TS_EC, COOKIE_TS_EC_REVOKED, COOKIE_TS_HEM}; +use crate::cookies::{CONSENT_COOKIE_NAMES, TS_IDENTITY_COOKIE_NAMES, handle_request_cookies}; use crate::ec::cookies::ec_id_has_only_allowed_chars; use crate::error::TrustedServerError; use crate::geo::GeoInfo; @@ -88,6 +89,7 @@ use crate::settings::Settings; use device::DeviceSignals; use self::finalize::BrowserSeed; +use self::identity::input::HashedEmail; use self::identity::measurement::{IdentityMeasurement, MeasurementInput, RouteKind}; use self::identity::request::{RequestIdentity, WriterDecision}; use self::identity::runtime::IdentityRuntime; @@ -216,6 +218,31 @@ fn parse_ec_from_request(req: &Request) -> Result, +) -> (Option, BTreeMap) { + let Some(jar) = handle_request_cookies(req).ok().flatten() else { + return (None, BTreeMap::new()); + }; + let hashed_email = jar + .get(COOKIE_TS_HEM) + .and_then(|cookie| HashedEmail::from_cookie_hex(cookie.value())); + let cookies = jar + .iter() + .filter(|cookie| { + !TS_IDENTITY_COOKIE_NAMES + .iter() + .chain(CONSENT_COOKIE_NAMES) + .any(|reserved| reserved.eq_ignore_ascii_case(cookie.name())) + }) + .map(|cookie| (cookie.name().to_owned(), cookie.value().to_owned())) + .collect(); + (hashed_email, cookies) +} + fn request_ec_id_if_allowed(value: &str, source: &str) -> Option { if ec_id_has_only_allowed_chars(value) { return Some(value.to_owned()); @@ -773,8 +800,9 @@ impl EcContext { /// Decides whether this request is the one browser-driven identity writer. /// /// Only an organic real-browser top-level document navigation that is not - /// a prefetch writes (spec 5.1). Also captures the user agent, `DNT` and - /// the page URL without query or fragment. + /// a prefetch writes (spec 5.1). Also captures the user agent, `DNT`, the + /// page URL without query or fragment, the hashed email from the `ts-hem` + /// cookie, and the first-party cookies identity providers may declare. /// /// The page URL is read from the request URI, so it needs an absolute /// URI. The Fastly adapter gives one (`edgezero-adapter-fastly` builds the @@ -806,12 +834,15 @@ impl EcContext { url.set_fragment(None); url }); + let (hashed_email, cookies) = provider_cookie_signals(req); self.identity.set_writer_decision(WriterDecision { writer, is_real_browser, user_agent, dnt, page_url, + hashed_email, + cookies, }); } @@ -829,6 +860,7 @@ impl EcContext { if !(same_origin && interval_sec > 0 && self.identity.is_real_browser()) { return; } + let (hashed_email, cookies) = provider_cookie_signals(req); self.identity.set_writer_decision(WriterDecision { writer: true, is_real_browser: true, @@ -843,6 +875,8 @@ impl EcContext { .and_then(|value| value.to_str().ok()) .is_some_and(|value| value.trim() == "1"), page_url: None, + hashed_email, + cookies, }); self.identity.set_spa_writer(true); } @@ -1019,6 +1053,8 @@ impl EcContext { user_agent: None, dnt: false, page_url: None, + hashed_email: None, + cookies: BTreeMap::new(), }); context } @@ -1562,6 +1598,71 @@ mod tests { assert!(!ec.ec_generated(), "should not mark as generated"); } + #[test] + fn provider_cookie_signals_read_the_hem_and_hide_identity_and_consent_cookies() { + let hem = "ab".repeat(32); + let cookie = format!( + "ts-hem={hem}; ts-ec=ec-value; ts-eids=eids; euconsent-v2=tc; __gpp=gpp; \ + us_privacy=1YNN; _lc2_fpi=fpi-example; other=value" + ); + let req = create_test_request(&[("cookie", &cookie)]); + + let (hashed_email, cookies) = provider_cookie_signals(&req); + + assert_eq!( + hashed_email.map(|email| email.to_hex()), + Some(hem), + "should read the ts-hem digest" + ); + let names: Vec<&str> = cookies.keys().map(String::as_str).collect(); + assert_eq!( + names, + ["_lc2_fpi", "other"], + "should offer only first-party cookies that are not TS identity or consent cookies" + ); + assert_eq!(cookies["_lc2_fpi"], "fpi-example", "should keep the value"); + } + + #[test] + fn provider_cookie_signals_hide_reserved_cookies_in_any_case() { + let req = create_test_request(&[("cookie", "TS-EC=x; Ts-Hem=y; EUCONSENT-V2=z; keep=1")]); + + let (_, cookies) = provider_cookie_signals(&req); + + let names: Vec<&str> = cookies.keys().map(String::as_str).collect(); + assert_eq!( + names, + ["keep"], + "should match reserved names case-insensitively" + ); + } + + #[test] + fn provider_cookie_signals_ignore_a_hem_that_is_not_a_digest() { + for value in ["", "person@example.com", &"zz".repeat(32), &"ab".repeat(31)] { + let cookie = format!("ts-hem={value}"); + let req = create_test_request(&[("cookie", &cookie)]); + + let (hashed_email, cookies) = provider_cookie_signals(&req); + + assert!( + hashed_email.is_none(), + "should not read `{value}` as a digest" + ); + assert!(cookies.is_empty(), "should never offer ts-hem as a cookie"); + } + } + + #[test] + fn provider_cookie_signals_are_empty_without_a_cookie_header() { + let req = create_test_request(&[]); + + let (hashed_email, cookies) = provider_cookie_signals(&req); + + assert!(hashed_email.is_none(), "should have no HEM"); + assert!(cookies.is_empty(), "should have no cookies"); + } + #[test] fn read_from_request_with_cookie_ec() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/integrations/connectid.rs b/crates/trusted-server-core/src/integrations/connectid.rs new file mode 100644 index 0000000000..8612ad22c7 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/connectid.rs @@ -0,0 +1,555 @@ +//! Yahoo `ConnectID` identity provider: resolves `ConnectID` server side. +//! +//! Calls Yahoo's user-profile sync endpoint with the hashed email from the +//! `ts-hem` cookie, the page and the consent strings. Yahoo returns the +//! `ConnectID` and a lifetime in hours. + +use std::time::Duration; + +use error_stack::Report; +use http::StatusCode; +use serde::Deserialize; +use validator::Validate; + +use crate::ec::identity::{ + ConsentPolicy, IdentityProvider, IdentityProviderConfig, IdentityProviderError, Input, + ProviderDescriptor, ProviderRequest, ProviderResponse, Resolution, ResolveInput, ResolvedEid, + ResolvedUid, Resolves, +}; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistration; +use crate::settings::{IntegrationConfig, Settings}; + +/// Integration ID of the `ConnectID` provider. +pub(crate) const CONNECTID_INTEGRATION_ID: &str = "connectid"; + +/// The consent purposes Yahoo's GVL entry lists (GVL version 182). +const YAHOO_TCF_PURPOSES: [u8; 11] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; + +/// Yahoo's IAB Global Vendor List ID. +const YAHOO_GVL_ID: u16 = 25; + +const SECONDS_PER_HOUR: u64 = 3_600; + +/// Config of the `connectid` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct ConnectIdConfig { + /// Enables the provider. + #[serde(default)] + pub enabled: bool, + /// The pixel ID Yahoo assigned to the publisher. + pub pixel_id: u64, + /// The sync endpoint; the pixel ID and `fed` are appended to its path. + #[serde(default = "default_endpoint")] + #[validate(url)] + pub endpoint: String, +} + +fn default_endpoint() -> String { + "https://ups.analytics.yahoo.com/ups".to_owned() +} + +impl IntegrationConfig for ConnectIdConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl IdentityProviderConfig for ConnectIdConfig { + fn build_descriptor(&self) -> Result> { + ProviderDescriptor::builder(CONNECTID_INTEGRATION_ID.parse()?) + .with_source("yahoo.com".parse()?) + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_needs([Input::HashedEmail]) + .with_wants([Input::PageUrl]), + ) + .with_consent_policy( + ConsentPolicy::strict() + .with_tcf_purposes(YAHOO_TCF_PURPOSES) + .with_tcf_vendor(YAHOO_GVL_ID), + ) + .build() + } +} + +/// The Yahoo `ConnectID` provider. +pub struct ConnectIdProvider { + descriptor: ProviderDescriptor, + pixel_id: String, +} + +impl ConnectIdProvider { + /// Builds the provider from its validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config cannot + /// describe a valid provider. + pub fn new(config: &ConnectIdConfig) -> Result> { + Ok(Self { + descriptor: config.build_descriptor()?, + pixel_id: config.pixel_id.to_string(), + }) + } +} + +#[derive(Deserialize)] +struct SyncReply { + #[serde(rename = "connectId")] + connect_id: Option, + /// The lowercase spelling Yahoo also sends, often next to `connectId`. + connectid: Option, + /// Hours. + ttl: Option, +} + +impl IdentityProvider for ConnectIdProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let mut url = input.endpoint()?.clone(); + url.path_segments_mut() + .map_err(|_| IdentityProviderError::config("endpoint cannot hold a path"))? + .pop_if_empty() + .push(&self.pixel_id) + .push("fed"); + let consent = input.consent(); + let mut request = ProviderRequest::get(&url) + .with_query("v", "1") + .with_query("1p", "0") + .with_query("pixelId", &self.pixel_id) + .with_query("he", &input.hashed_email()?.to_hex()) + .with_query("gdpr", if consent.gdpr_applies() { "1" } else { "0" }); + if let Some(tc_string) = consent.raw_tc_string() { + request = request.with_query("gdpr_consent", tc_string); + } + if let Some(us_privacy) = consent.raw_us_privacy() { + request = request.with_query("us_privacy", us_privacy); + } + if let Some(gpp) = consent.raw_gpp_string() { + request = request.with_query("gpp", gpp); + } + if let Some(ids) = consent.gpp_section_ids() { + let ids: Vec = ids.iter().map(u16::to_string).collect(); + request = request.with_query("gpp_sid", &ids.join(",")); + } + if let Ok(page) = input.page_url() { + request = request.with_query("url", page.as_str()); + } + Ok(Some(request)) + } + + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + if response.status() == StatusCode::NO_CONTENT { + return Ok(Resolution::no_match()); + } + let reply: SyncReply = response.success_json()?; + let Some(connect_id) = [reply.connect_id, reply.connectid] + .into_iter() + .flatten() + .find(|id| !id.is_empty()) + else { + return Ok(Resolution::no_match()); + }; + let mut eid = ResolvedEid::new( + input.primary_source(), + ResolvedUid::new(connect_id).with_atype(3), + ); + if let Some(hours) = reply.ttl.filter(|hours| *hours > 0) { + eid = eid.with_refresh_in(Duration::from_secs(hours.saturating_mul(SECONDS_PER_HOUR))); + } + Ok(Resolution::found([eid])) + } +} + +/// Registers the `ConnectID` provider when `[integrations.connectid]` is +/// enabled. +/// +/// # Errors +/// +/// Returns an error when the config does not parse or describes an invalid +/// provider. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.integration_config::(CONNECTID_INTEGRATION_ID)? + else { + return Ok(None); + }; + log::info!( + "Registering Yahoo ConnectID identity provider (pixel={})", + config.pixel_id + ); + Ok(Some( + IntegrationRegistration::builder(CONNECTID_INTEGRATION_ID) + .try_with_identity_provider(ConnectIdProvider::new(&config))? + .without_js() + .build(), + )) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::ec::identity::test_support::{ + ConformanceCases, HarnessSetup, ProviderHarness, TestConsent, + check_descriptor_ignores_secrets, + }; + use crate::ec::identity::{HashedEmail, ResolveKind, When}; + + fn config() -> ConnectIdConfig { + ConnectIdConfig { + enabled: true, + pixel_id: 58_776, + endpoint: "https://ups.example.com/ups".to_owned(), + } + } + + fn provider() -> ConnectIdProvider { + ConnectIdProvider::new(&config()).expect("should build the provider") + } + + #[test] + fn sends_the_hashed_email_to_the_pixel() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_hashed_email(HashedEmail::for_test([0xab; 32])) + .with_page_url("https://www.example.com/article"); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + assert_eq!( + request.url().path(), + "/ups/58776/fed", + "should address the pixel" + ); + let he = request + .url() + .query_pairs() + .find(|(key, _)| key == "he") + .map(|(_, value)| value.into_owned()); + assert_eq!(he, Some("ab".repeat(32)), "should send the HEM hex"); + } + + #[test] + fn resolves_the_connect_id_with_its_ttl() { + let provider = provider(); + let mut harness = + ProviderHarness::new(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "connectId": "cid-example", "connectid": "cid-example", "ttl": 24 }), + ) + .expect("should parse"); + + let eid = &resolution.eids()[0]; + assert_eq!(eid.source().as_str(), "yahoo.com", "should use yahoo.com"); + assert_eq!(eid.uids()[0].id(), "cid-example", "should keep the ID"); + assert!(eid.refresh().is_some(), "should set a refresh from the ttl"); + } + + #[test] + fn an_empty_reply_is_no_match() { + let provider = provider(); + let mut harness = + ProviderHarness::new(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let resolution = harness + .respond(StatusCode::OK, &json!({})) + .expect("should parse"); + + assert!(resolution.is_no_match(), "should be no match"); + } + + fn with_email(provider: &ConnectIdProvider) -> ProviderHarness<'_> { + ProviderHarness::new(provider).with_hashed_email(HashedEmail::for_test([0x0f; 32])) + } + + fn requested(harness: &mut ProviderHarness<'_>) -> ProviderRequest { + harness + .request() + .expect("should build a request") + .expect("should not skip") + } + + fn query_values(request: &ProviderRequest, name: &str) -> Vec { + request + .url() + .query_pairs() + .filter(|(key, _)| key == name) + .map(|(_, value)| value.into_owned()) + .collect() + } + + #[test] + fn skips_without_a_hashed_email() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + + assert!( + harness.request().expect("should not fail").is_none(), + "should not call Yahoo without the HEM it needs" + ); + } + + #[test] + fn skips_without_vendor_consent_where_gdpr_applies() { + let provider = provider(); + let mut harness = + with_email(&provider).with_consent(TestConsent::default().with_gdpr_applies(true)); + + assert!( + harness.request().expect("should not fail").is_none(), + "should not call Yahoo without vendor 25" + ); + } + + #[test] + fn addresses_the_pixel_and_sends_lowercase_hex() { + let provider = provider(); + let mut harness = with_email(&provider); + + let request = requested(&mut harness); + + assert_eq!( + request.url().path(), + "/ups/58776/fed", + "should address the pixel" + ); + assert_eq!( + query_values(&request, "pixelId"), + ["58776"], + "should repeat the pixel" + ); + assert_eq!( + query_values(&request, "he"), + ["0f".repeat(32)], + "should send lowercase hex" + ); + assert_eq!( + query_values(&request, "1p"), + ["0"], + "should mark a server call" + ); + assert_eq!(query_values(&request, "gdpr"), ["0"], "should send gdpr"); + } + + #[test] + fn sends_consent_strings_it_has_and_nothing_else() { + let provider = provider(); + let mut harness = + with_email(&provider).with_consent(TestConsent::default().with_us_privacy("1YNN")); + + let request = requested(&mut harness); + + assert_eq!( + query_values(&request, "us_privacy"), + ["1YNN"], + "should send US privacy" + ); + for name in ["gdpr_consent", "gpp", "gpp_sid", "url"] { + assert!( + query_values(&request, name).is_empty(), + "should not send `{name}`" + ); + } + } + + #[test] + fn sends_the_page_without_its_query() { + let provider = provider(); + let mut harness = with_email(&provider) + .with_page_url("https://www.example.com/a/b?email=x%40example.com"); + + let request = requested(&mut harness); + + assert_eq!( + query_values(&request, "url"), + ["https://www.example.com/a/b"], + "should never forward the page query" + ); + } + + #[test] + fn accepts_either_key_and_prefers_connect_id() { + for (reply, expected) in [ + (json!({ "connectId": "camel" }), "camel"), + (json!({ "connectid": "lower" }), "lower"), + ( + json!({ "connectId": "camel", "connectid": "lower" }), + "camel", + ), + (json!({ "connectId": "", "connectid": "lower" }), "lower"), + ] { + let provider = provider(); + let mut harness = with_email(&provider); + requested(&mut harness); + + let resolution = harness + .respond(StatusCode::OK, &reply) + .expect("should parse"); + + assert_eq!( + resolution.eids()[0].uids()[0].id(), + expected, + "should pick `{expected}` from {reply}" + ); + } + } + + #[test] + fn the_id_is_a_person_based_atype_3_on_yahoo_com() { + let provider = provider(); + let mut harness = with_email(&provider); + requested(&mut harness); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "connectId": "c1" })) + .expect("should parse"); + + let eid = &resolution.eids()[0]; + assert_eq!(eid.source().as_str(), "yahoo.com", "should write yahoo.com"); + assert_eq!(eid.uids()[0].atype(), Some(3), "should be atype 3"); + } + + #[test] + fn a_zero_or_missing_ttl_sets_no_refresh() { + for reply in [ + json!({ "connectId": "c1", "ttl": 0 }), + json!({ "connectId": "c1" }), + ] { + let provider = provider(); + let mut harness = with_email(&provider); + requested(&mut harness); + + let resolution = harness + .respond(StatusCode::OK, &reply) + .expect("should parse"); + + assert_eq!( + resolution.eids()[0].refresh(), + None, + "should not refresh for {reply}" + ); + } + } + + #[test] + fn a_huge_ttl_saturates_instead_of_overflowing() { + let provider = provider(); + let mut harness = with_email(&provider); + requested(&mut harness); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "connectId": "c1", "ttl": u64::MAX }), + ) + .expect("should parse"); + + assert_eq!( + resolution.eids()[0].refresh(), + Some(When::In(Duration::from_secs(u64::MAX))), + "should saturate" + ); + } + + #[test] + fn no_content_and_null_ids_are_no_match() { + let provider = provider(); + let mut harness = with_email(&provider); + requested(&mut harness); + let resolution = harness + .respond_raw(StatusCode::NO_CONTENT, &[], b"") + .expect("should accept 204"); + assert!(resolution.is_no_match(), "204 should be no match"); + + let mut harness = with_email(&provider); + requested(&mut harness); + let resolution = harness + .respond(StatusCode::OK, &json!({ "connectId": null, "ttl": 24 })) + .expect("should parse"); + assert!(resolution.is_no_match(), "a null ID should be no match"); + } + + #[test] + fn rejects_errors_and_malformed_bodies() { + let provider = provider(); + let mut harness = with_email(&provider); + requested(&mut harness); + let error = harness + .respond(StatusCode::FORBIDDEN, &json!({})) + .expect_err("should reject 403"); + assert!( + matches!( + error.current_context(), + IdentityProviderError::Rejected { .. } + ), + "should be rejected: {error:?}" + ); + + let mut harness = with_email(&provider); + requested(&mut harness); + let error = harness + .respond_raw(StatusCode::OK, &[], b"{\"connectId\": 5}") + .expect_err("should fail on a numeric ID"); + assert!( + matches!(error.current_context(), IdentityProviderError::Malformed), + "should be malformed: {error:?}" + ); + } + + #[test] + fn declares_the_hem_as_needed() { + let descriptor = config().build_descriptor().expect("should build"); + + let resolves = descriptor.resolves().expect("should resolve"); + assert!( + resolves + .needs(ResolveKind::Initial) + .contains(&Input::HashedEmail), + "should need the HEM" + ); + assert_eq!( + descriptor.consent_policy().tcf_vendor(), + Some(YAHOO_GVL_ID), + "should name GVL 25" + ); + } + + #[test] + fn conforms() { + ProviderHarness::new(&provider()).check_conformance( + &ConformanceCases::default().with_resolve_case( + HarnessSetup::default().with_hashed_email(HashedEmail::for_test([0xab; 32])), + StatusCode::OK, + &json!({ "connectId": "cid-example" }), + ), + ); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets(&config(), &config()); + } +} diff --git a/crates/trusted-server-core/src/integrations/id5.rs b/crates/trusted-server-core/src/integrations/id5.rs new file mode 100644 index 0000000000..2340e7d716 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/id5.rs @@ -0,0 +1,858 @@ +//! ID5 identity provider: resolves the ID5 ID server side. +//! +//! Calls ID5's fetch endpoint with the request's consent, page and network +//! signals, and the hashed email when the JS SDK set one. ID5 returns either +//! complete EIDs under `ids` or the legacy `universal_uid`. The signature ID5 +//! returns is kept as private state and sent on the next call. +//! +//! The request field names for the client IP and user agent follow ID5's +//! server-to-server integration and should be confirmed with ID5 for each +//! partner account. + +use std::collections::BTreeMap; +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use error_stack::Report; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use url::Url; +use validator::Validate; + +use crate::ec::identity::{ + ConsentPolicy, IdentityProvider, IdentityProviderConfig, IdentityProviderError, Input, + ProviderDescriptor, ProviderRequest, ProviderResponse, Resolution, ResolveInput, ResolvedEid, + ResolvedUid, Resolves, +}; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistration; +use crate::integrations::vendor_eid::VendorEid; +use crate::integrations::vendor_request::is_routable_ip; +use crate::settings::{IntegrationConfig, Settings}; + +/// Integration ID of the ID5 provider. +pub(crate) const ID5_INTEGRATION_ID: &str = "id5"; + +/// The consent purposes ID5's GVL entry lists (GVL version 182). +const ID5_TCF_PURPOSES: [u8; 4] = [1, 3, 5, 10]; + +/// ID5's IAB Global Vendor List ID. +const ID5_GVL_ID: u16 = 131; + +/// The sources ID5 can return. Operators narrow them with +/// `[ec.identity.providers.id5].sources`. +const ID5_SOURCES: &[&str] = &["id5-sync.com", "true-link-id5-sync.com", "euid.eu"]; + +/// The universal UID ID5 returns for a visitor who opted out at ID5. +const ID5_OPTED_OUT_UID: &str = "0"; + +/// Config of the `id5` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct Id5Config { + /// Enables the provider. + #[serde(default)] + pub enabled: bool, + /// The partner number ID5 assigned to the publisher. + pub partner_id: u32, + /// The fetch endpoint; `.json` is appended to its path. + #[serde(default = "default_endpoint")] + #[validate(url)] + pub endpoint: String, +} + +fn default_endpoint() -> String { + "https://id5-sync.com/g/v2".to_owned() +} + +impl IntegrationConfig for Id5Config { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl IdentityProviderConfig for Id5Config { + fn build_descriptor(&self) -> Result> { + let mut builder = ProviderDescriptor::builder(ID5_INTEGRATION_ID.parse()?); + for source in ID5_SOURCES { + builder = builder.with_source(source.parse()?); + } + builder + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_wants([ + Input::ClientIp, + Input::UserAgent, + Input::PageUrl, + Input::HashedEmail, + ]) + .with_private_state(), + ) + .with_consent_policy( + ConsentPolicy::strict() + .with_tcf_purposes(ID5_TCF_PURPOSES) + .with_tcf_vendor(ID5_GVL_ID), + ) + .build() + } +} + +/// The ID5 provider. +pub struct Id5Provider { + descriptor: ProviderDescriptor, + partner_id: u32, +} + +impl Id5Provider { + /// Builds the provider from its validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config cannot + /// describe a valid provider. + pub fn new(config: &Id5Config) -> Result> { + Ok(Self { + descriptor: config.build_descriptor()?, + partner_id: config.partner_id, + }) + } +} + +/// The signature ID5 returns, sent back on the next call. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct Id5State { + signature: String, +} + +#[derive(Serialize)] +struct FetchBody<'a> { + partner: u32, + v: &'static str, + o: &'static str, + #[serde(rename = "nbPage")] + nb_page: u32, + gdpr: u8, + #[serde(skip_serializing_if = "Option::is_none")] + gdpr_consent: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + us_privacy: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + gpp_string: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + gpp_sid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + s: Option, + #[serde(skip_serializing_if = "Option::is_none")] + u: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + tml: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pd: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ua: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + ip: Option, +} + +#[derive(Deserialize)] +struct FetchReply { + universal_uid: Option, + signature: Option, + #[serde(default)] + ext: Option, + #[serde(default)] + ids: BTreeMap, + cache_control: Option, +} + +#[derive(Deserialize)] +struct Id5Id { + eid: Option, +} + +#[derive(Deserialize)] +struct CacheControl { + max_age_sec: Option, +} + +impl IdentityProvider for Id5Provider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let consent = input.consent(); + let page = input.page_url().ok().map(Url::as_str); + let body = FetchBody { + partner: self.partner_id, + v: env!("CARGO_PKG_VERSION"), + o: "trusted-server", + nb_page: 1, + gdpr: u8::from(consent.gdpr_applies()), + gdpr_consent: consent.raw_tc_string(), + us_privacy: consent.raw_us_privacy(), + gpp_string: consent.raw_gpp_string(), + gpp_sid: consent.gpp_section_ids().map(join_ids), + s: input.state::()?.map(|state| state.signature), + u: page, + tml: page, + pd: input + .hashed_email() + .ok() + .map(|email| STANDARD.encode(format!("1={}", email.to_hex()))), + ua: input.user_agent().ok(), + ip: input + .client_ip() + .ok() + .filter(|ip| is_routable_ip(*ip)) + .map(|ip| ip.to_string()), + }; + let mut url = input.endpoint()?.clone(); + url.path_segments_mut() + .map_err(|_| IdentityProviderError::config("endpoint cannot hold a path"))? + .pop_if_empty() + .push(&format!("{}.json", self.partner_id)); + ProviderRequest::post_json(&url, &body)? + .try_with_header("origin", &format!("https://{}", input.publisher_domain())) + .map(Some) + } + + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let reply: FetchReply = response.success_json()?; + let primary = input.primary_source(); + + let mut eids: Vec = reply + .ids + .values() + .filter_map(|id| { + let eid = id.eid.as_ref()?; + let source = self.descriptor.source_named(&eid.source)?; + eid.to_resolved(source) + }) + .collect(); + if !eids.iter().any(|eid| eid.source() == primary) { + match reply.universal_uid.as_deref() { + Some(ID5_OPTED_OUT_UID) => return Ok(Resolution::invalidate(primary)), + Some(uid) if !uid.is_empty() => { + let mut resolved = ResolvedUid::new(uid).with_atype(1); + if let Some(link_type) = reply.ext.as_ref().and_then(|ext| ext.get("linkType")) + { + resolved = resolved.with_ext(json!({ "linkType": link_type })); + } + eids.push(ResolvedEid::new(primary, resolved)); + } + _ => {} + } + } + if let Some(max_age) = reply.cache_control.and_then(|cache| cache.max_age_sec) { + eids = eids + .into_iter() + .map(|eid| eid.with_refresh_in(Duration::from_secs(max_age))) + .collect(); + } + + let mut resolution = if eids.is_empty() { + Resolution::no_match() + } else { + let returned: Vec<_> = eids.iter().map(|eid| eid.source().clone()).collect(); + let mut found = Resolution::found(eids); + for (source, _) in input.sources() { + if !returned.contains(source) && input.stored(source).is_some() { + found = found.also_invalidate(source); + } + } + found + }; + if let Some(signature) = reply.signature { + resolution = resolution.replacing_state(&Id5State { signature }); + } + Ok(resolution) + } +} + +fn join_ids(ids: &[u16]) -> String { + ids.iter().map(u16::to_string).collect::>().join(",") +} + +/// Registers the ID5 provider when `[integrations.id5]` is enabled. +/// +/// # Errors +/// +/// Returns an error when the config does not parse or describes an invalid +/// provider. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.integration_config::(ID5_INTEGRATION_ID)? else { + return Ok(None); + }; + log::info!( + "Registering ID5 identity provider (partner={})", + config.partner_id + ); + Ok(Some( + IntegrationRegistration::builder(ID5_INTEGRATION_ID) + .try_with_identity_provider(Id5Provider::new(&config))? + .without_js() + .build(), + )) +} + +#[cfg(test)] +mod tests { + use http::StatusCode; + + use super::*; + use crate::ec::identity::test_support::{ + ConformanceCases, HarnessSetup, ProviderHarness, TestConsent, + check_descriptor_ignores_secrets, + }; + use crate::ec::identity::{HashedEmail, SourceDomain, StateChange, StoredId, When}; + + fn config() -> Id5Config { + Id5Config { + enabled: true, + partner_id: 1234, + endpoint: default_endpoint(), + } + } + + fn provider() -> Id5Provider { + Id5Provider::new(&config()).expect("should build the provider") + } + + #[test] + fn sends_consent_page_and_hashed_email() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_user_agent("ExampleBrowser/1.0") + .with_page_url("https://www.example.com/article") + .with_hashed_email(HashedEmail::for_test([0xab; 32])) + .with_consent(TestConsent::default().with_us_privacy("1YNN")) + .with_state(&Id5State { + signature: "prior-signature".to_owned(), + }); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + assert_eq!( + request.url().as_str(), + "https://id5-sync.com/g/v2/1234.json", + "should call the partner's fetch URL" + ); + let body: serde_json::Value = harness.request_json(); + + assert_eq!(body["partner"], 1234, "should send the partner"); + assert_eq!(body["gdpr"], 0, "should send that GDPR does not apply"); + assert_eq!(body["us_privacy"], "1YNN", "should send US privacy"); + assert_eq!( + body["s"], "prior-signature", + "should send the stored signature" + ); + assert_eq!( + body["u"], "https://www.example.com/article", + "should send the page" + ); + assert_eq!(body["ip"], "192.0.2.10", "should send the client IP"); + assert_eq!( + body["pd"], + STANDARD.encode(format!("1={}", "ab".repeat(32))), + "should send the hashed email as partner data" + ); + } + + #[test] + fn resolves_complete_eids_and_keeps_the_signature() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ + "signature": "next-signature", + "ids": { + "id5": { "eid": { + "source": "id5-sync.com", + "uids": [{ "id": "ID5*example", "atype": 1, "ext": { "linkType": 2 } }] + }}, + "trueLinkId": { "eid": { + "source": "true-link-id5-sync.com", + "uids": [{ "id": "true-link-example", "atype": 1 }] + }}, + "other": { "eid": { + "source": "other.example.com", + "uids": [{ "id": "ignored" }] + }} + }, + "cache_control": { "max_age_sec": 7200 } + }), + ) + .expect("should parse"); + + let sources: Vec<&str> = resolution + .eids() + .iter() + .map(|eid| eid.source().as_str()) + .collect(); + assert_eq!( + sources, + ["id5-sync.com", "true-link-id5-sync.com"], + "should keep only declared sources" + ); + let state: Id5State = harness.state_of(&resolution).expect("should replace state"); + assert_eq!( + state.signature, "next-signature", + "should keep the signature" + ); + } + + #[test] + fn falls_back_to_the_universal_uid() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "universal_uid": "ID5*legacy", "ext": { "linkType": 1 } }), + ) + .expect("should parse"); + + let eid = &resolution.eids()[0]; + assert_eq!( + eid.source().as_str(), + "id5-sync.com", + "should use the primary source" + ); + assert_eq!(eid.uids()[0].id(), "ID5*legacy", "should keep the UID"); + assert_eq!( + eid.uids()[0].ext(), + Some(&json!({ "linkType": 1 })), + "should keep the link type" + ); + } + + #[test] + fn an_opted_out_visitor_invalidates_the_id() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + harness.request().expect("should build a request"); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "universal_uid": "0" })) + .expect("should parse"); + + assert_eq!( + resolution.invalidated()[0].as_str(), + "id5-sync.com", + "should invalidate the ID5 ID" + ); + } + + fn source(domain: &str) -> SourceDomain { + domain.parse().expect("should parse the source") + } + + fn requested(harness: &mut ProviderHarness<'_>) -> ProviderRequest { + harness + .request() + .expect("should build a request") + .expect("should not skip") + } + + #[test] + fn leaves_out_every_signal_the_request_lacks() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let body: serde_json::Value = harness.request_json(); + + for key in [ + "s", + "pd", + "ua", + "ip", + "u", + "tml", + "gdpr_consent", + "gpp_string", + ] { + assert!(body.get(key).is_none(), "should not send `{key}`: {body}"); + } + assert_eq!(body["o"], "trusted-server", "should name the origin"); + assert_eq!(body["nbPage"], 1, "should send a page count"); + } + + #[test] + fn leaves_out_a_loopback_or_private_client_ip() { + for ip in ["127.0.0.1", "10.0.0.8", "::1"] { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider).with_client_ip(ip); + requested(&mut harness); + + let body: serde_json::Value = harness.request_json(); + + assert!(body.get("ip").is_none(), "should not send {ip}"); + } + } + + #[test] + fn sends_an_ipv6_client_ip() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider).with_client_ip("2001:db8::7"); + requested(&mut harness); + + let body: serde_json::Value = harness.request_json(); + + assert_eq!( + body["ip"], "2001:db8::7", + "should send a public IPv6 address" + ); + } + + #[test] + fn skips_without_vendor_consent_where_gdpr_applies() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_consent(TestConsent::default().with_gdpr_applies(true)); + + let request = harness.request().expect("should not fail"); + + assert!(request.is_none(), "should not call ID5 without vendor 131"); + } + + #[test] + fn sends_the_origin_of_the_publisher() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + + let request = requested(&mut harness); + + let origin = request + .headers() + .get("origin") + .and_then(|value| value.to_str().ok()) + .expect("should send an origin"); + assert!( + origin.starts_with("https://") && !origin.ends_with('/'), + "should send an https origin without a path: {origin}" + ); + } + + #[test] + fn appends_the_partner_file_to_an_endpoint_with_a_trailing_slash() { + let provider = Id5Provider::new(&Id5Config { + endpoint: "https://id5.example.com/custom/".to_owned(), + ..config() + }) + .expect("should build the provider"); + let mut harness = ProviderHarness::new(&provider); + + let request = requested(&mut harness); + + assert_eq!( + request.url().as_str(), + "https://id5.example.com/custom/1234.json", + "should not double the slash" + ); + } + + #[test] + fn rejects_a_non_success_status() { + for status in [StatusCode::BAD_REQUEST, StatusCode::INTERNAL_SERVER_ERROR] { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let error = harness + .respond(status, &json!({ "universal_uid": "ID5*x" })) + .expect_err("should reject the status"); + + assert!( + matches!(error.current_context(), IdentityProviderError::Rejected { status: got } if *got == status), + "should report {status} as rejected: {error:?}" + ); + } + } + + #[test] + fn a_body_that_is_not_json_is_malformed() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let error = harness + .respond_raw(StatusCode::OK, &[], b"") + .expect_err("should fail"); + + assert!( + matches!(error.current_context(), IdentityProviderError::Malformed), + "should be malformed: {error:?}" + ); + } + + #[test] + fn an_empty_reply_is_no_match() { + for reply in [ + json!({}), + json!({ "universal_uid": "" }), + json!({ "ids": {} }), + ] { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond(StatusCode::OK, &reply) + .expect("should parse"); + + assert!(resolution.is_no_match(), "should be no match for {reply}"); + } + } + + #[test] + fn an_ids_entry_without_uids_falls_back_to_the_universal_uid() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ + "universal_uid": "ID5*legacy", + "ids": { "id5": { "eid": { "source": "id5-sync.com", "uids": [{ "id": "" }] } } } + }), + ) + .expect("should parse"); + + assert_eq!(resolution.eids().len(), 1, "should resolve one entry"); + assert_eq!( + resolution.eids()[0].uids()[0].id(), + "ID5*legacy", + "should use the universal UID" + ); + } + + #[test] + fn a_complete_primary_eid_wins_over_an_opted_out_universal_uid() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ + "universal_uid": "0", + "ids": { "id5": { "eid": { "source": "id5-sync.com", "uids": [{ "id": "ID5*new" }] } } } + }), + ) + .expect("should parse"); + + assert!(resolution.invalidated().is_empty(), "should not invalidate"); + assert_eq!( + resolution.eids()[0].uids()[0].id(), + "ID5*new", + "should keep the EID" + ); + } + + #[test] + fn cache_control_sets_the_refresh_of_every_entry() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "universal_uid": "ID5*x", "cache_control": { "max_age_sec": 7200 } }), + ) + .expect("should parse"); + + assert_eq!( + resolution.eids()[0].refresh(), + Some(When::In(Duration::from_secs(7200))), + "should refresh after max_age_sec" + ); + } + + #[test] + fn without_cache_control_sets_no_refresh() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "universal_uid": "ID5*x" })) + .expect("should parse"); + + assert_eq!( + resolution.eids()[0].refresh(), + None, + "should leave refresh to defaults" + ); + } + + #[test] + fn a_stored_source_missing_from_the_reply_is_invalidated() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider).with_stored(StoredId::for_test( + &source("true-link-id5-sync.com"), + "old-true-link", + )); + requested(&mut harness); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "universal_uid": "ID5*x" })) + .expect("should parse"); + + assert_eq!( + resolution.invalidated(), + [source("true-link-id5-sync.com")], + "should remove the stale TrueLink ID" + ); + } + + #[test] + fn a_source_neither_stored_nor_returned_is_not_invalidated() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "universal_uid": "ID5*x" })) + .expect("should parse"); + + assert!( + resolution.invalidated().is_empty(), + "should invalidate nothing" + ); + } + + #[test] + fn keeps_a_new_signature_even_without_an_id() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "signature": "fresh" })) + .expect("should parse"); + + assert!(resolution.is_no_match(), "should find nothing"); + let state: Id5State = harness.state_of(&resolution).expect("should replace state"); + assert_eq!(state.signature, "fresh", "should keep the signature"); + } + + #[test] + fn keeps_the_stored_signature_when_the_reply_has_none() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider).with_state(&Id5State { + signature: "prior".to_owned(), + }); + requested(&mut harness); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "universal_uid": "ID5*x" })) + .expect("should parse"); + + assert!( + matches!(resolution.state_change(), StateChange::Keep), + "should leave the state alone" + ); + } + + #[test] + fn unknown_reply_fields_are_ignored() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "universal_uid": "ID5*x", "privacy": { "jurisdiction": "other" }, "future": [1] }), + ) + .expect("should parse a reply with new fields"); + + assert_eq!(resolution.eids().len(), 1, "should still resolve"); + } + + #[test] + fn descriptor_declares_the_vendor_sources_and_inputs() { + let descriptor = config().build_descriptor().expect("should build"); + + let sources: Vec<&str> = descriptor + .sources() + .iter() + .map(SourceDomain::as_str) + .collect(); + assert_eq!(sources, ID5_SOURCES, "should declare every ID5 source"); + let resolves = descriptor.resolves().expect("should resolve"); + assert!( + resolves.wants().contains(&Input::HashedEmail), + "should want the hashed email" + ); + assert!( + resolves.has_private_state(), + "should keep the signature as state" + ); + } + + #[test] + fn config_requires_a_partner_id_and_a_url_endpoint() { + let missing: Result = serde_json::from_value(json!({ "enabled": true })); + assert!(missing.is_err(), "should require partner_id"); + + let config: Id5Config = serde_json::from_value( + json!({ "enabled": true, "partner_id": 1, "endpoint": "not a url" }), + ) + .expect("should parse"); + assert!( + config.validate().is_err(), + "should reject a non-URL endpoint" + ); + + let defaulted: Id5Config = + serde_json::from_value(json!({ "partner_id": 1 })).expect("should parse"); + assert!(!defaulted.is_enabled(), "should default to disabled"); + assert_eq!( + defaulted.endpoint, + default_endpoint(), + "should default the endpoint" + ); + } + + #[test] + fn conforms() { + ProviderHarness::new(&provider()).check_conformance_with_state::( + &ConformanceCases::default().with_resolve_case( + HarnessSetup::default().with_client_ip("192.0.2.10"), + StatusCode::OK, + &json!({ "universal_uid": "ID5*example", "signature": "s" }), + ), + ); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets(&config(), &config()); + } +} diff --git a/crates/trusted-server-core/src/integrations/liveintent.rs b/crates/trusted-server-core/src/integrations/liveintent.rs new file mode 100644 index 0000000000..fb7f115eb0 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/liveintent.rs @@ -0,0 +1,901 @@ +//! `LiveIntent` identity provider: resolves `LiveIntent`'s IDs server side. +//! +//! Calls `LiveIntent`'s identity exchange with the visitor's first-party +//! `LiveIntent` cookie, consent and network signals, and asks for the +//! attributes the operator configured. Each attribute `LiveIntent` returns +//! maps to one `OpenRTB` source. + +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use error_stack::Report; +use http::StatusCode; +use serde::Deserialize; +use serde_json::{Map, Value, json}; +use validator::Validate; + +use crate::ec::identity::{ + ConsentPolicy, IdentityProvider, IdentityProviderConfig, IdentityProviderError, Input, + ProviderDescriptor, ProviderRequest, ProviderResponse, Resolution, ResolveInput, ResolvedEid, + ResolvedUid, Resolves, SourceDomain, +}; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistration; +use crate::integrations::vendor_request::is_routable_ip; +use crate::settings::{IntegrationConfig, Settings}; + +/// Integration ID of the `LiveIntent` provider. +pub(crate) const LIVEINTENT_INTEGRATION_ID: &str = "liveintent"; + +/// The consent purposes `LiveIntent`'s GVL entry lists (GVL version 182). +const LIVEINTENT_TCF_PURPOSES: [u8; 3] = [1, 2, 7]; + +/// `LiveIntent`'s IAB Global Vendor List ID. +const LIVEINTENT_GVL_ID: u16 = 148; + +/// `LiveIntent`'s first-party ID cookie on the publisher's domain. +const FIRST_PARTY_COOKIE: &str = "_lc2_fpi"; + +/// The attribute that carries `LiveIntent`'s own ID. +const NON_ID_ATTRIBUTE: &str = "nonId"; + +/// Each attribute `LiveIntent` resolves and the source it carries. +const ATTRIBUTE_SOURCES: &[(&str, &str)] = &[ + (NON_ID_ATTRIBUTE, "liveintent.com"), + ("bidswitch", "bidswitch.net"), + ("medianet", "media.net"), + ("magnite", "rubiconproject.com"), + ("index", "liveintent.indexexchange.com"), + ("openx", "openx.net"), + ("pubmatic", "pubmatic.com"), + ("sovrn", "liveintent.sovrn.com"), + ("sharethrough", "sharethrough.com"), + ("sonobi", "liveintent.sonobi.com"), + ("triplelift", "liveintent.triplelift.com"), + ("zetassp", "zeta-ssp.liveintent.com"), + ("vidazoo", "liveintent.vidazoo.com"), + ("nexxen", "liveintent.unrulymedia.com"), + ("uid2", "uidapi.com"), + ("thetradedesk", "adserver.org"), +]; + +/// Config of the `liveintent` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct LiveIntentConfig { + /// Enables the provider. + #[serde(default)] + pub enabled: bool, + /// The publisher ID `LiveIntent` assigned, or `any`. + #[serde(default = "default_publisher_id")] + #[validate(length(min = 1))] + pub publisher_id: String, + /// The distributor ID `LiveIntent` assigned to a distributor of the + /// integration, if any. It takes the place of `prebid` in the path. + pub distributor_id: Option, + /// The identity exchange endpoint. + #[serde(default = "default_endpoint")] + #[validate(url)] + pub endpoint: String, + /// The attributes to resolve, from the table in this module. Each one's + /// source needs an `[[ec.partners]]` entry. Default: `nonId`. + #[serde(default = "default_resolve")] + #[validate(length(min = 1))] + pub resolve: Vec, +} + +fn default_publisher_id() -> String { + "any".to_owned() +} + +fn default_endpoint() -> String { + "https://idx.liadm.com/idex".to_owned() +} + +fn default_resolve() -> Vec { + vec![NON_ID_ATTRIBUTE.to_owned()] +} + +impl IntegrationConfig for LiveIntentConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl LiveIntentConfig { + /// The configured attributes and their sources, in config order. + fn attributes(&self) -> Result, Report> { + self.resolve + .iter() + .map(|attribute| { + let (name, source) = ATTRIBUTE_SOURCES + .iter() + .find(|(name, _)| name == attribute) + .ok_or_else(|| { + IdentityProviderError::config(format!( + "unknown LiveIntent attribute `{attribute}`" + )) + })?; + Ok(((*name).to_owned(), source.parse()?)) + }) + .collect() + } +} + +impl IdentityProviderConfig for LiveIntentConfig { + fn build_descriptor(&self) -> Result> { + let mut builder = ProviderDescriptor::builder(LIVEINTENT_INTEGRATION_ID.parse()?); + for (_, source) in self.attributes()? { + builder = builder.with_source(source); + } + builder + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_wants([Input::ClientIp, Input::PageUrl]) + .with_cookie(FIRST_PARTY_COOKIE), + ) + .with_consent_policy( + ConsentPolicy::strict() + .with_tcf_purposes(LIVEINTENT_TCF_PURPOSES) + .with_tcf_vendor(LIVEINTENT_GVL_ID), + ) + .build() + } +} + +/// The `LiveIntent` provider. +pub struct LiveIntentProvider { + descriptor: ProviderDescriptor, + path: String, + distributor_id: Option, + attributes: Vec<(String, SourceDomain)>, +} + +impl LiveIntentProvider { + /// Builds the provider from its validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config names an + /// unknown attribute or cannot describe a valid provider. + pub fn new(config: &LiveIntentConfig) -> Result> { + let distributor = config.distributor_id.as_deref().unwrap_or("prebid"); + Ok(Self { + descriptor: config.build_descriptor()?, + path: format!("{distributor}/{}", config.publisher_id), + distributor_id: config.distributor_id.clone(), + attributes: config.attributes()?, + }) + } +} + +impl IdentityProvider for LiveIntentProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let mut url = input.endpoint()?.clone(); + url.path_segments_mut() + .map_err(|_| IdentityProviderError::config("endpoint cannot hold a path"))? + .pop_if_empty() + .extend(self.path.split('/')); + let mut request = ProviderRequest::get(&url) + .try_with_header("origin", &format!("https://{}", input.publisher_domain()))?; + if let Some(distributor) = &self.distributor_id { + request = request.with_query("did", distributor); + } + if let Some(fpid) = input.cookie(FIRST_PARTY_COOKIE)? { + request = request.with_query("duid", fpid); + } + let consent = input.consent(); + request = request.with_query("gdpr", if consent.gdpr_applies() { "1" } else { "0" }); + if let Some(tc_string) = consent.raw_tc_string() { + request = request.with_query("gdpr_consent", tc_string); + } + if let Some(us_privacy) = consent.raw_us_privacy() { + request = request.with_query("us_privacy", us_privacy); + } + if let Some(gpp) = consent.raw_gpp_string() { + request = request.with_query("gpp_s", gpp); + } + if let Some(ids) = consent.gpp_section_ids() { + let ids: Vec = ids.iter().map(u16::to_string).collect(); + request = request.with_query("gpp_as", &ids.join(",")); + } + if let Ok(page) = input.page_url() { + request = request.with_query("pu", page.origin().ascii_serialization().as_str()); + } + if let Ok(ip) = input.client_ip() + && is_routable_ip(ip) + { + let name = if ip.is_ipv4() { "pip" } else { "pip6" }; + request = request.with_query(name, &URL_SAFE_NO_PAD.encode(ip.to_string())); + } + for (attribute, _) in &self.attributes { + request = request.with_query("resolve", attribute); + } + Ok(Some(request)) + } + + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + if response.status() == StatusCode::NO_CONTENT { + return Ok(Resolution::no_match()); + } + let reply: Map = response.success_json()?; + let segments = reply.get("segments").cloned(); + let lifetime = expires_in(response, input); + + let eids: Vec = self + .attributes + .iter() + .filter(|(_, source)| input.sources().any(|(wanted, _)| wanted == source)) + .filter_map(|(attribute, source)| { + let id = reply.get(attribute)?.as_str().filter(|id| !id.is_empty())?; + let mut eid = ResolvedEid::new(source, resolved_uid(attribute, id)); + if attribute == NON_ID_ATTRIBUTE + && let Some(segments) = &segments + { + eid = eid.with_ext(json!({ "segments": segments })); + } + if attribute == "thetradedesk" { + eid = eid.with_matcher("liveintent.com"); + } + if let Some(lifetime) = lifetime { + eid = eid.with_expires_in(lifetime); + } + Some(eid) + }) + .collect(); + Ok(if eids.is_empty() { + Resolution::no_match() + } else { + Resolution::found(eids) + }) + } +} + +fn resolved_uid(attribute: &str, id: &str) -> ResolvedUid { + let uid = ResolvedUid::new(id).with_atype(3); + match attribute { + NON_ID_ATTRIBUTE => uid, + "thetradedesk" => { + uid.with_ext(json!({ "rtiPartner": "TDID", "provider": "liveintent.com" })) + } + _ => uid.with_ext(json!({ "provider": "liveintent.com" })), + } +} + +/// The lifetime `LiveIntent` gives in its `Expires` header, when it is a +/// valid HTTP date in the future. +fn expires_in(response: &ProviderResponse, input: &ResolveInput<'_>) -> Option { + let expires = response.header("expires")?; + let at = chrono::DateTime::parse_from_rfc2822(expires) + .ok()? + .timestamp(); + let seconds = u64::try_from(at).ok()?.checked_sub(input.now().as_secs())?; + (seconds > 0).then(|| Duration::from_secs(seconds)) +} + +/// Registers the `LiveIntent` provider when `[integrations.liveintent]` is +/// enabled. +/// +/// # Errors +/// +/// Returns an error when the config does not parse or describes an invalid +/// provider. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = + settings.integration_config::(LIVEINTENT_INTEGRATION_ID)? + else { + return Ok(None); + }; + log::info!( + "Registering LiveIntent identity provider (attributes={})", + config.resolve.join(",") + ); + Ok(Some( + IntegrationRegistration::builder(LIVEINTENT_INTEGRATION_ID) + .try_with_identity_provider(LiveIntentProvider::new(&config))? + .without_js() + .build(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::identity::test_support::{ + ConformanceCases, HarnessSetup, ProviderHarness, TestConsent, + check_descriptor_ignores_secrets, + }; + use crate::ec::identity::{UnixTime, When}; + + fn config(resolve: &[&str]) -> LiveIntentConfig { + LiveIntentConfig { + enabled: true, + publisher_id: "12345".to_owned(), + distributor_id: None, + endpoint: default_endpoint(), + resolve: resolve.iter().map(|name| (*name).to_owned()).collect(), + } + } + + fn provider(resolve: &[&str]) -> LiveIntentProvider { + LiveIntentProvider::new(&config(resolve)).expect("should build the provider") + } + + #[test] + fn calls_the_exchange_with_cookie_consent_and_attributes() { + let provider = provider(&["nonId", "thetradedesk"]); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_page_url("https://www.example.com/article?x=1") + .with_cookie(FIRST_PARTY_COOKIE, "fpi-example") + .with_consent(TestConsent::default().with_us_privacy("1YNN")); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + let url = request.url(); + assert_eq!( + url.path(), + "/idex/prebid/12345", + "should address the publisher" + ); + assert_eq!( + request + .headers() + .get("origin") + .map(http::HeaderValue::as_bytes), + Some(b"https://www.example.com".as_slice()), + "should send the publisher origin, which LiveIntent requires" + ); + let query: Vec<(String, String)> = url.query_pairs().into_owned().collect(); + let value = |name: &str| { + query + .iter() + .filter(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + .collect::>() + }; + assert_eq!( + value("duid"), + ["fpi-example"], + "should send the first-party ID" + ); + assert_eq!(value("us_privacy"), ["1YNN"], "should send US privacy"); + assert_eq!( + value("pu"), + ["https://www.example.com"], + "should send the page origin" + ); + assert_eq!( + value("pip"), + [URL_SAFE_NO_PAD.encode("192.0.2.10")], + "should send the encoded IP" + ); + assert_eq!( + value("resolve"), + ["nonId", "thetradedesk"], + "should ask for each configured attribute" + ); + } + + #[test] + fn leaves_out_a_loopback_client_ip() { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider).with_client_ip("127.0.0.1"); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + assert!( + !request + .url() + .query_pairs() + .any(|(key, _)| key == "pip" || key == "pip6"), + "should not send an address LiveIntent cannot match" + ); + } + + #[test] + fn maps_attributes_to_sources() { + let provider = provider(&["nonId", "thetradedesk", "pubmatic"]); + let mut harness = ProviderHarness::new(&provider); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ + "nonId": "li-example", + "segments": ["s1"], + "thetradedesk": "tdid-example", + "bidswitch": "not-configured" + }), + ) + .expect("should parse"); + + let eids = resolution.eids(); + assert_eq!( + eids.len(), + 2, + "should map only configured attributes it returned" + ); + assert_eq!( + eids[0].source().as_str(), + "liveintent.com", + "should map nonId" + ); + assert_eq!( + eids[0].ext(), + Some(&json!({ "segments": ["s1"] })), + "should keep segments" + ); + assert_eq!( + eids[1].source().as_str(), + "adserver.org", + "should map the TDID" + ); + assert_eq!( + eids[1].matcher(), + Some("liveintent.com"), + "should name the matcher" + ); + } + + #[test] + fn no_content_is_no_match() { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider); + harness.request().expect("should build a request"); + + let resolution = harness + .respond_raw(StatusCode::NO_CONTENT, &[], b"") + .expect("should accept 204"); + + assert!(resolution.is_no_match(), "should be no match"); + } + + #[test] + fn rejects_an_unknown_attribute() { + assert!( + LiveIntentProvider::new(&config(&["unknown"])).is_err(), + "should refuse an attribute outside the table" + ); + } + + /// 2026-01-01T00:00:00Z. + const NOW: u64 = 1_767_225_600; + + fn requested(harness: &mut ProviderHarness<'_>) -> ProviderRequest { + harness + .request() + .expect("should build a request") + .expect("should not skip") + } + + fn query_values(request: &ProviderRequest, name: &str) -> Vec { + request + .url() + .query_pairs() + .filter(|(key, _)| key == name) + .map(|(_, value)| value.into_owned()) + .collect() + } + + #[test] + fn sends_the_distributor_in_the_path_and_as_did_only_when_configured() { + let distributed = LiveIntentProvider::new(&LiveIntentConfig { + distributor_id: Some("did-0000".to_owned()), + ..config(&["nonId"]) + }) + .expect("should build the provider"); + let mut harness = ProviderHarness::new(&distributed); + + let request = requested(&mut harness); + + assert_eq!( + request.url().path(), + "/idex/did-0000/12345", + "should address the distributor" + ); + assert_eq!( + query_values(&request, "did"), + ["did-0000"], + "should send did" + ); + + let plain = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&plain); + let request = requested(&mut harness); + assert!( + query_values(&request, "did").is_empty(), + "should not send did without a distributor, which LiveIntent rejects" + ); + } + + #[test] + fn defaults_the_publisher_to_any() { + let config: LiveIntentConfig = + serde_json::from_value(json!({ "enabled": true })).expect("should parse"); + let provider = LiveIntentProvider::new(&config).expect("should build"); + let mut harness = ProviderHarness::new(&provider); + + let request = requested(&mut harness); + + assert_eq!( + request.url().path(), + "/idex/prebid/any", + "should use the defaults" + ); + assert_eq!( + query_values(&request, "resolve"), + ["nonId"], + "should ask for nonId only" + ); + } + + #[test] + fn leaves_out_signals_the_request_lacks() { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider); + + let request = requested(&mut harness); + + for name in [ + "duid", + "pu", + "pip", + "pip6", + "gdpr_consent", + "us_privacy", + "gpp_s", + "gpp_as", + ] { + assert!( + query_values(&request, name).is_empty(), + "should not send `{name}`" + ); + } + assert_eq!( + query_values(&request, "gdpr"), + ["0"], + "should always send gdpr" + ); + } + + #[test] + fn sends_an_ipv6_client_ip_as_pip6() { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider).with_client_ip("2001:db8::7"); + + let request = requested(&mut harness); + + assert_eq!( + query_values(&request, "pip6"), + [URL_SAFE_NO_PAD.encode("2001:db8::7")], + "should send the encoded IPv6 address" + ); + assert!( + query_values(&request, "pip").is_empty(), + "should not send pip" + ); + } + + #[test] + fn leaves_out_private_and_mapped_loopback_addresses() { + for ip in ["10.1.2.3", "192.168.0.4", "fd00::1", "::ffff:127.0.0.1"] { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider).with_client_ip(ip); + + let request = requested(&mut harness); + + assert!( + query_values(&request, "pip").is_empty() + && query_values(&request, "pip6").is_empty(), + "should not send {ip}" + ); + } + } + + #[test] + fn sends_only_the_page_origin_never_its_path_or_query() { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider) + .with_page_url("https://www.example.com:8443/private/path?email=a%40example.com#x"); + + let request = requested(&mut harness); + + assert_eq!( + query_values(&request, "pu"), + ["https://www.example.com:8443"], + "should send the origin only" + ); + } + + #[test] + fn skips_without_vendor_consent_where_gdpr_applies() { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider) + .with_consent(TestConsent::default().with_gdpr_applies(true)); + + assert!( + harness.request().expect("should not fail").is_none(), + "should not call LiveIntent without vendor 148" + ); + } + + #[test] + fn partner_ids_carry_the_provider_and_atype_3() { + let provider = provider(&["nonId", "pubmatic", "uid2"]); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "nonId": "li-1", "pubmatic": "pm-1", "uid2": "uid2-1" }), + ) + .expect("should parse"); + + let eids = resolution.eids(); + assert_eq!(eids.len(), 3, "should map all three"); + assert_eq!( + eids[0].uids()[0].ext(), + None, + "nonId should carry no provider ext" + ); + for eid in &eids[1..] { + let uid = &eid.uids()[0]; + assert_eq!( + uid.atype(), + Some(3), + "should mark {} as atype 3", + eid.source() + ); + assert_eq!( + uid.ext(), + Some(&json!({ "provider": "liveintent.com" })), + "should name LiveIntent as provider for {}", + eid.source() + ); + } + assert_eq!( + eids[2].source().as_str(), + "uidapi.com", + "should map uid2 to uidapi.com" + ); + } + + #[test] + fn the_tdid_carries_the_rti_partner() { + let provider = provider(&["thetradedesk"]); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "thetradedesk": "tdid-1" })) + .expect("should parse"); + + assert_eq!( + resolution.eids()[0].uids()[0].ext(), + Some(&json!({ "rtiPartner": "TDID", "provider": "liveintent.com" })), + "should match Prebid's TDID shape" + ); + } + + #[test] + fn ignores_empty_and_non_string_values() { + let provider = provider(&["nonId", "pubmatic", "openx"]); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "nonId": "", "pubmatic": 42, "openx": null }), + ) + .expect("should parse"); + + assert!(resolution.is_no_match(), "should find nothing usable"); + } + + #[test] + fn segments_attach_only_to_the_non_id() { + let provider = provider(&["pubmatic"]); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "pubmatic": "pm-1", "segments": ["s1"] }), + ) + .expect("should parse"); + + assert_eq!( + resolution.eids()[0].ext(), + None, + "should not put segments on a partner ID" + ); + } + + #[test] + fn a_future_expires_header_sets_the_lifetime() { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider).with_now(UnixTime::from_secs(NOW)); + requested(&mut harness); + + let resolution = harness + .respond_raw( + StatusCode::OK, + &[("expires", "Fri, 02 Jan 2026 00:00:00 GMT")], + br#"{"nonId":"li-1"}"#, + ) + .expect("should parse"); + + assert_eq!( + resolution.eids()[0].expiry(), + Some(When::In(Duration::from_secs(86_400))), + "should expire when LiveIntent says" + ); + } + + #[test] + fn a_past_or_invalid_expires_header_sets_no_lifetime() { + for expires in [ + "Wed, 31 Dec 2025 00:00:00 GMT", + "Thu, 01 Jan 2026 00:00:00 GMT", + "soon", + "0", + ] { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider).with_now(UnixTime::from_secs(NOW)); + requested(&mut harness); + + let resolution = harness + .respond_raw( + StatusCode::OK, + &[("expires", expires)], + br#"{"nonId":"li-1"}"#, + ) + .expect("should parse"); + + assert_eq!( + resolution.eids()[0].expiry(), + None, + "should ignore `{expires}`" + ); + } + } + + #[test] + fn rejects_a_non_success_status_other_than_204() { + for status in [ + StatusCode::BAD_REQUEST, + StatusCode::TOO_MANY_REQUESTS, + StatusCode::BAD_GATEWAY, + ] { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let error = harness + .respond(status, &json!({ "message": "Malformed did" })) + .expect_err("should reject"); + + assert!( + matches!(error.current_context(), IdentityProviderError::Rejected { status: got } if *got == status), + "should report {status}: {error:?}" + ); + } + } + + #[test] + fn a_body_that_is_not_a_json_object_is_malformed() { + for body in [&b"[]"[..], b"\"x\"", b"not json"] { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider); + requested(&mut harness); + + let error = harness + .respond_raw(StatusCode::OK, &[], body) + .expect_err("should fail"); + + assert!( + matches!(error.current_context(), IdentityProviderError::Malformed), + "should be malformed for {:?}: {error:?}", + String::from_utf8_lossy(body) + ); + } + } + + #[test] + fn declares_one_source_per_attribute_and_the_first_party_cookie() { + let descriptor = config(&["nonId", "medianet", "nexxen"]) + .build_descriptor() + .expect("should build"); + + let sources: Vec<&str> = descriptor + .sources() + .iter() + .map(SourceDomain::as_str) + .collect(); + assert_eq!( + sources, + ["liveintent.com", "media.net", "liveintent.unrulymedia.com"], + "should declare the configured attributes' sources in order" + ); + let resolves = descriptor.resolves().expect("should resolve"); + assert!( + resolves.cookies().contains(FIRST_PARTY_COOKIE), + "should declare _lc2_fpi" + ); + assert!( + !resolves.wants().contains(&Input::HashedEmail), + "should not want the HEM" + ); + } + + #[test] + fn every_attribute_in_the_table_builds() { + let names: Vec<&str> = ATTRIBUTE_SOURCES.iter().map(|(name, _)| *name).collect(); + + let descriptor = config(&names) + .build_descriptor() + .expect("should build every attribute"); + + assert_eq!( + descriptor.sources().len(), + ATTRIBUTE_SOURCES.len(), + "one source each" + ); + } + + #[test] + fn config_rejects_an_empty_resolve_list_and_a_bad_endpoint() { + let empty = LiveIntentConfig { + resolve: Vec::new(), + ..config(&["nonId"]) + }; + assert!(empty.validate().is_err(), "should require an attribute"); + + let bad = LiveIntentConfig { + endpoint: "nope".to_owned(), + ..config(&["nonId"]) + }; + assert!(bad.validate().is_err(), "should require a URL endpoint"); + } + + #[test] + fn conforms() { + ProviderHarness::new(&provider(&["nonId"])).check_conformance( + &ConformanceCases::default().with_resolve_case( + HarnessSetup::default().with_cookie(FIRST_PARTY_COOKIE, "fpi-example"), + StatusCode::OK, + &json!({ "nonId": "li-example" }), + ), + ); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets(&config(&["nonId"]), &config(&["nonId"])); + } +} diff --git a/crates/trusted-server-core/src/integrations/liveramp.rs b/crates/trusted-server-core/src/integrations/liveramp.rs new file mode 100644 index 0000000000..bbfce4f030 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/liveramp.rs @@ -0,0 +1,677 @@ +//! `LiveRamp` ATS identity provider: resolves and refreshes ATS envelopes +//! server side. +//! +//! An initial lookup sends the hashed email from the `ts-hem` cookie. A +//! refresh sends the stored envelope. Envelopes expire after 15 days in +//! California and 30 days elsewhere. The response shape and the consent +//! parameter names are assumed (spec 12.1) and should be confirmed with +//! `LiveRamp`. + +use std::time::Duration; + +use error_stack::Report; +use http::StatusCode; +use serde::{Deserialize, Serialize}; +use validator::Validate; + +use crate::ec::identity::{ + ConsentPolicy, IdentityProvider, IdentityProviderConfig, IdentityProviderError, Input, + ProviderDescriptor, ProviderRequest, ProviderResponse, Resolution, ResolveInput, ResolveKind, + ResolvedEid, ResolvedUid, Resolves, +}; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistration; +use crate::integrations::vendor_request::is_routable_ip; +use crate::settings::{IntegrationConfig, Settings}; + +/// Integration ID of the `LiveRamp` provider. +pub(crate) const LIVERAMP_INTEGRATION_ID: &str = "liveramp"; + +/// The consent purposes `LiveRamp`'s GVL entry lists (GVL version 182). +const LIVERAMP_TCF_PURPOSES: [u8; 10] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + +/// `LiveRamp`'s IAB Global Vendor List ID. +const LIVERAMP_GVL_ID: u16 = 97; + +/// The regular ATS envelope product. +const ATS_ENVELOPE_TYPE: u32 = 19; +const ATS_ENVELOPE_SOURCE: &str = "envelopeLiveramp"; + +/// Identifier types: 4 is a hashed email, 19 an envelope refresh. +const IT_HASHED_EMAIL: &str = "4"; +const IT_ENVELOPE: &str = "19"; + +/// Consent type 4: an IAB TCF v2 string. +const CT_TCF: &str = "4"; + +const SECONDS_PER_DAY: u64 = 86_400; + +/// Config of the `liveramp` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct LiveRampConfig { + /// Enables the provider. + #[serde(default)] + pub enabled: bool, + /// The ATS envelope endpoint. + #[serde(default = "default_endpoint")] + #[validate(url)] + pub endpoint: String, + /// `LiveRamp` placement ID. An operator identifier, not a secret. + #[validate(length(min = 1))] + pub placement_id: String, + /// Countries the placement is approved for, as ISO 3166-1 alpha-2. + #[validate(length(min = 1))] + pub approved_countries: Vec, +} + +fn default_endpoint() -> String { + "https://api.rlcdn.com/api/identity/v2/envelope".to_owned() +} + +impl IntegrationConfig for LiveRampConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl IdentityProviderConfig for LiveRampConfig { + fn build_descriptor(&self) -> Result> { + ProviderDescriptor::builder(LIVERAMP_INTEGRATION_ID.parse()?) + .with_source("liveramp.com".parse()?) + .with_expiring_ids() + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_needs([Input::ClientIp, Input::Region]) + .with_needs_for(ResolveKind::Initial, [Input::HashedEmail]), + ) + .with_consent_policy( + ConsentPolicy::strict() + .with_tcf_purposes(LIVERAMP_TCF_PURPOSES) + .with_tcf_vendor(LIVERAMP_GVL_ID), + ) + .build() + } +} + +/// The `LiveRamp` ATS provider. +pub struct LiveRampProvider { + descriptor: ProviderDescriptor, + placement_id: String, + approved_countries: Vec, +} + +impl LiveRampProvider { + /// Builds the provider from its validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config cannot + /// describe a valid provider. + pub fn new(config: &LiveRampConfig) -> Result> { + Ok(Self { + descriptor: config.build_descriptor()?, + placement_id: config.placement_id.clone(), + approved_countries: config + .approved_countries + .iter() + .map(|country| country.to_ascii_uppercase()) + .collect(), + }) + } +} + +/// Whether the call was an initial lookup or a refresh, carried from +/// `request` to `response`: a 204 means different things for each. +#[derive(Debug, Serialize, Deserialize)] +enum LiveRampCall { + Initial, + Refresh, +} + +#[derive(Deserialize)] +struct EnvelopeReply { + #[serde(default)] + envelopes: Vec, +} + +#[derive(Deserialize)] +struct EnvelopeProduct { + #[serde(rename = "type")] + product_type: u32, + source: String, + value: String, +} + +impl IdentityProvider for LiveRampProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let region = input.region()?; + if !self + .approved_countries + .iter() + .any(|country| country == region.country()) + { + return Ok(None); + } + let source = input.primary_source(); + let (call, identifier_type, identifier_value) = match input.stored(source) { + Some(stored) if input.is_refresh(source) => { + (LiveRampCall::Refresh, IT_ENVELOPE, stored.uid().to_owned()) + } + _ => ( + LiveRampCall::Initial, + IT_HASHED_EMAIL, + input.hashed_email()?.to_hex(), + ), + }; + let consent = input.consent(); + let mut request = ProviderRequest::get(input.endpoint()?) + .with_query("pid", &self.placement_id) + .with_query("it", identifier_type) + .with_query("iv", &identifier_value) + .with_query("atype", "3") + .try_with_call_context(&call)? + .try_with_header("origin", &format!("https://{}", input.publisher_domain()))?; + let client_ip = input.client_ip()?; + if is_routable_ip(client_ip) { + request = request.try_with_header("x-forwarded-for", &client_ip.to_string())?; + } + if let Some(tc_string) = consent.raw_tc_string() { + request = request.with_query("ct", CT_TCF).with_query("cv", tc_string); + } + if let Some(gpp) = consent.raw_gpp_string() { + request = request.with_query("gpp", gpp); + } + if let Some(ids) = consent.gpp_section_ids() { + let ids: Vec = ids.iter().map(u16::to_string).collect(); + request = request.with_query("gpp_sid", &ids.join(",")); + } + Ok(Some(request)) + } + + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let source = input.primary_source(); + let call: LiveRampCall = response + .call_context()? + .ok_or_else(|| IdentityProviderError::config("missing call context"))?; + if response.status() == StatusCode::NO_CONTENT { + return Ok(match call { + LiveRampCall::Refresh => Resolution::invalidate(source), + LiveRampCall::Initial => Resolution::no_match(), + }); + } + let reply: EnvelopeReply = response.success_json()?; + let Some(envelope) = reply.envelopes.into_iter().find(|product| { + product.product_type == ATS_ENVELOPE_TYPE + && product.source == ATS_ENVELOPE_SOURCE + && !product.value.is_empty() + }) else { + return Ok(Resolution::no_match()); + }; + let lifetime_days = if input.region()?.is("US", "CA") { + 15 + } else { + 30 + }; + let eid = ResolvedEid::new(source, ResolvedUid::new(envelope.value).with_atype(3)) + .with_expires_in(Duration::from_secs(lifetime_days * SECONDS_PER_DAY)); + Ok(Resolution::found([eid])) + } +} + +/// Registers the `LiveRamp` provider when `[integrations.liveramp]` is +/// enabled. +/// +/// # Errors +/// +/// Returns an error when the config does not parse or describes an invalid +/// provider. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.integration_config::(LIVERAMP_INTEGRATION_ID)? + else { + return Ok(None); + }; + log::info!( + "Registering LiveRamp ATS identity provider (countries={})", + config.approved_countries.join(",") + ); + Ok(Some( + IntegrationRegistration::builder(LIVERAMP_INTEGRATION_ID) + .try_with_identity_provider(LiveRampProvider::new(&config))? + .without_js() + .build(), + )) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::ec::identity::test_support::{ + ConformanceCases, HarnessSetup, ProviderHarness, TestConsent, + check_descriptor_ignores_secrets, + }; + use crate::ec::identity::{HashedEmail, ResolveKind, SourceDomain, StoredId}; + + fn config() -> LiveRampConfig { + LiveRampConfig { + enabled: true, + endpoint: "https://ats.example.com/api/identity/v2/envelope".to_owned(), + placement_id: "placement-1".to_owned(), + approved_countries: vec!["us".to_owned()], + } + } + + fn provider() -> LiveRampProvider { + LiveRampProvider::new(&config()).expect("should build the provider") + } + + fn source() -> SourceDomain { + "liveramp.com".parse().expect("should parse the source") + } + + fn query(request: &ProviderRequest, name: &str) -> Option { + request + .url() + .query_pairs() + .find(|(key, _)| key == name) + .map(|(_, value)| value.into_owned()) + } + + #[test] + fn initial_lookup_sends_the_hashed_email() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_region("US", Some("NY")) + .with_hashed_email(HashedEmail::for_test([0xab; 32])); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + assert_eq!( + query(&request, "it").as_deref(), + Some("4"), + "should be a HEM lookup" + ); + assert_eq!( + query(&request, "iv"), + Some("ab".repeat(32)), + "should send the HEM hex" + ); + assert_eq!( + request + .headers() + .get("x-forwarded-for") + .map(http::HeaderValue::as_bytes), + Some(b"192.0.2.10".as_slice()), + "should forward the client IP" + ); + } + + #[test] + fn skips_unapproved_countries() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_region("FR", None) + .with_hashed_email(HashedEmail::for_test([0xab; 32])); + + assert!( + harness.request().expect("should not fail").is_none(), + "should not call LiveRamp outside approved countries" + ); + } + + #[test] + fn california_envelopes_expire_after_15_days() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_region("US", Some("CA")) + .with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "envelopes": [ + { "type": 19, "source": "envelopeLiveramp", "value": "envelope-example" } + ]}), + ) + .expect("should parse"); + + let eid = &resolution.eids()[0]; + assert_eq!( + eid.uids()[0].id(), + "envelope-example", + "should keep the envelope" + ); + assert_eq!( + eid.expiry().map(|when| harness.resolve(when)), + Some(crate::ec::identity::UnixTime::from_secs( + 1_760_000_000 + 15 * SECONDS_PER_DAY + )), + "should expire after 15 days in California" + ); + } + + #[test] + fn a_refused_refresh_invalidates_the_envelope() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_region("US", Some("NY")) + .with_stored( + StoredId::for_test(&source(), "old-envelope") + .expiring_at(crate::ec::identity::UnixTime::from_secs(1)), + ); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + assert_eq!( + query(&request, "it").as_deref(), + Some("19"), + "should be a refresh" + ); + + let resolution = harness + .respond_raw(StatusCode::NO_CONTENT, &[], b"") + .expect("should accept 204"); + assert_eq!( + resolution.invalidated(), + [source()], + "should invalidate the envelope" + ); + } + + fn us_visitor(provider: &LiveRampProvider) -> ProviderHarness<'_> { + ProviderHarness::new(provider) + .with_client_ip("192.0.2.10") + .with_region("US", Some("NY")) + } + + fn envelope_reply(value: &str) -> serde_json::Value { + json!({ "envelopes": [{ "type": 19, "source": "envelopeLiveramp", "value": value }] }) + } + + #[test] + fn forwards_only_a_routable_client_ip() { + for (ip, forwarded) in [ + ("192.0.2.10", true), + ("127.0.0.1", false), + ("10.0.0.3", false), + ] { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip(ip) + .with_region("US", Some("NY")) + .with_hashed_email(HashedEmail::for_test([0x01; 32])); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + assert_eq!( + request.headers().get("x-forwarded-for").is_some(), + forwarded, + "should forward {ip} only when it is routable" + ); + } + } + + #[test] + fn an_initial_lookup_without_a_hashed_email_is_skipped() { + let provider = provider(); + let mut harness = us_visitor(&provider); + + assert!( + harness.request().expect("should not fail").is_none(), + "should not call LiveRamp without a HEM or an envelope" + ); + } + + #[test] + fn a_refresh_needs_no_hashed_email_and_sends_the_stored_envelope() { + let provider = provider(); + let mut harness = us_visitor(&provider).with_stored( + StoredId::for_test(&source(), "stored-envelope") + .expiring_at(crate::ec::identity::UnixTime::from_secs(1)), + ); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + assert_eq!( + query(&request, "it").as_deref(), + Some("19"), + "should refresh" + ); + assert_eq!( + query(&request, "iv").as_deref(), + Some("stored-envelope"), + "should send the envelope" + ); + } + + #[test] + fn sends_the_placement_atype_origin_and_no_cookie() { + let provider = provider(); + let mut harness = + us_visitor(&provider).with_hashed_email(HashedEmail::for_test([0x01; 32])); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + assert_eq!( + query(&request, "pid").as_deref(), + Some("placement-1"), + "should send the placement" + ); + assert_eq!( + query(&request, "atype").as_deref(), + Some("3"), + "should ask for atype 3" + ); + assert!( + request.headers().get("origin").is_some(), + "should send an origin" + ); + assert!( + request.headers().get(http::header::COOKIE).is_none(), + "should send no cookie" + ); + for name in ["ct", "cv", "gpp", "gpp_sid"] { + assert!( + query(&request, name).is_none(), + "should not send `{name}` without consent strings" + ); + } + } + + #[test] + fn an_undecodable_gpp_string_fails_closed() { + let provider = provider(); + let mut harness = us_visitor(&provider) + .with_hashed_email(HashedEmail::for_test([0x01; 32])) + .with_consent(TestConsent::default().with_gpp_string("not-gpp", &[7])); + + assert!( + harness.request().expect("should not fail").is_none(), + "should not send a HEM under consent it cannot read" + ); + } + + #[test] + fn envelopes_outside_california_expire_after_30_days() { + let provider = provider(); + let mut harness = + us_visitor(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let resolution = harness + .respond(StatusCode::OK, &envelope_reply("envelope-ny")) + .expect("should parse"); + + let eid = &resolution.eids()[0]; + assert_eq!( + eid.expiry().map(|when| harness.resolve(when)), + Some(crate::ec::identity::UnixTime::from_secs( + 1_760_000_000 + 30 * SECONDS_PER_DAY + )), + "should expire after 30 days" + ); + assert_eq!(eid.uids()[0].atype(), Some(3), "should be atype 3"); + } + + #[test] + fn picks_the_ats_envelope_among_other_products() { + let provider = provider(); + let mut harness = + us_visitor(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "envelopes": [ + { "type": 20, "source": "envelopeLiveramp", "value": "wrong-type" }, + { "type": 19, "source": "other", "value": "wrong-source" }, + { "type": 19, "source": "envelopeLiveramp", "value": "" }, + { "type": 19, "source": "envelopeLiveramp", "value": "the-envelope" } + ]}), + ) + .expect("should parse"); + + assert_eq!(resolution.eids().len(), 1, "should keep one envelope"); + assert_eq!( + resolution.eids()[0].uids()[0].id(), + "the-envelope", + "should pick type 19" + ); + } + + #[test] + fn no_usable_envelope_is_no_match() { + for reply in [json!({}), json!({ "envelopes": [] }), envelope_reply("")] { + let provider = provider(); + let mut harness = + us_visitor(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let resolution = harness + .respond(StatusCode::OK, &reply) + .expect("should parse"); + + assert!(resolution.is_no_match(), "should be no match for {reply}"); + } + } + + #[test] + fn a_204_on_an_initial_lookup_is_no_match_not_an_invalidation() { + let provider = provider(); + let mut harness = + us_visitor(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let resolution = harness + .respond_raw(StatusCode::NO_CONTENT, &[], b"") + .expect("should accept 204"); + + assert!(resolution.is_no_match(), "should be no match"); + assert!(resolution.invalidated().is_empty(), "should not invalidate"); + } + + #[test] + fn rejects_errors_and_malformed_envelopes() { + let provider = provider(); + let mut harness = + us_visitor(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + let error = harness + .respond(StatusCode::INTERNAL_SERVER_ERROR, &json!({})) + .expect_err("should reject 500"); + assert!( + matches!( + error.current_context(), + IdentityProviderError::Rejected { .. } + ), + "should be rejected: {error:?}" + ); + + let mut harness = + us_visitor(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + let error = harness + .respond(StatusCode::OK, &json!({ "envelopes": [{ "type": "19" }] })) + .expect_err("should fail on a product without fields"); + assert!( + matches!(error.current_context(), IdentityProviderError::Malformed), + "should be malformed: {error:?}" + ); + } + + #[test] + fn declares_an_expiring_source_and_needs_the_hem_only_initially() { + let descriptor = config().build_descriptor().expect("should build"); + + assert!(descriptor.is_expiring(&source()), "envelopes should expire"); + let resolves = descriptor.resolves().expect("should resolve"); + assert!( + resolves + .needs(ResolveKind::Initial) + .contains(&Input::HashedEmail), + "an initial lookup should need the HEM" + ); + assert!( + !resolves + .needs(ResolveKind::Refresh) + .contains(&Input::HashedEmail), + "a refresh should not need the HEM" + ); + } + + #[test] + fn conforms() { + ProviderHarness::new(&provider()).check_conformance( + &ConformanceCases::default().with_resolve_case( + HarnessSetup::default() + .with_client_ip("192.0.2.10") + .with_region("US", Some("NY")) + .with_hashed_email(HashedEmail::for_test([0xab; 32])), + StatusCode::OK, + &json!({ "envelopes": [ + { "type": 19, "source": "envelopeLiveramp", "value": "envelope-example" } + ]}), + ), + ); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets(&config(), &config()); + } +} diff --git a/crates/trusted-server-core/src/integrations/lockr.rs b/crates/trusted-server-core/src/integrations/lockr.rs index 2f86d30e40..2bcf3404f4 100644 --- a/crates/trusted-server-core/src/integrations/lockr.rs +++ b/crates/trusted-server-core/src/integrations/lockr.rs @@ -19,6 +19,7 @@ use validator::Validate; use crate::constants::INTERNAL_HEADERS; use crate::error::TrustedServerError; +use crate::integrations::lockr_identity::LockrIdentityProvider; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, @@ -28,7 +29,7 @@ use crate::integrations::{ use crate::platform::{PlatformHttpRequest, RuntimeServices}; use crate::settings::{IntegrationConfig, Settings}; -const LOCKR_INTEGRATION_ID: &str = "lockr"; +pub(crate) const LOCKR_INTEGRATION_ID: &str = "lockr"; /// Configuration for Lockr integration. #[derive(Debug, Deserialize, Validate)] @@ -71,6 +72,12 @@ pub struct LockrConfig { #[serde(default)] #[validate(url)] pub origin_override: Option, + + /// Registers Lockr as a capture-only identity provider that revokes IDs + /// when Lockr's API reports a consent revocation. Needs + /// `[ec.identity.providers.lockr].sources`. Default: off. + #[serde(default)] + pub capture_identity: bool, } impl IntegrationConfig for LockrConfig { @@ -336,12 +343,14 @@ pub fn register( integration.config.rewrite_sdk ); - Ok(Some( - IntegrationRegistration::builder(LOCKR_INTEGRATION_ID) - .with_proxy(integration.clone()) - .with_attribute_rewriter(integration) - .build(), - )) + let mut builder = IntegrationRegistration::builder(LOCKR_INTEGRATION_ID) + .with_proxy(integration.clone()) + .with_attribute_rewriter(integration.clone()); + if integration.config.capture_identity { + builder = + builder.try_with_identity_provider(LockrIdentityProvider::new(&integration.config))?; + } + Ok(Some(builder.build())) } #[async_trait(?Send)] @@ -450,6 +459,7 @@ mod tests { rewrite_sdk: true, rewrite_sdk_host: None, origin_override: None, + capture_identity: false, } } diff --git a/crates/trusted-server-core/src/integrations/lockr_identity.rs b/crates/trusted-server-core/src/integrations/lockr_identity.rs new file mode 100644 index 0000000000..7bacd615d1 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/lockr_identity.rs @@ -0,0 +1,433 @@ +//! Lockr as a capture-only identity provider (spec 12.2). +//! +//! Reads the Lockr API responses Trusted Server already proxies. Capture is +//! revoke-only: a `revoke-consent` call opts the visitor out of Lockr's +//! sources, and a later token issue clears that opt-out. The tokens it +//! reads are not stored; browser ingest carries Lockr's IDs to KV. + +use std::collections::BTreeMap; + +use error_stack::Report; +use http::Method; +use serde::Deserialize; + +use crate::ec::identity::{ + CaptureInput, Captures, EidProvenance, IdentityProvider, IdentityProviderConfig, + IdentityProviderError, ProviderDescriptor, Resolution, ResolvedEid, UnixTime, +}; +use crate::integrations::lockr::{LOCKR_INTEGRATION_ID, LockrConfig}; +use crate::integrations::vendor_eid::VendorEid; + +/// The proxied Lockr API prefix. +const LOCKR_API: &str = "/integrations/lockr/api/publisher/app/v2/identityLockr"; + +/// The Lockr API calls captured, by their last path segment. +const CAPTURED_ENDPOINTS: &[&str] = &[ + "page-view", + "generate-tokens", + "refresh-tokens", + "sync-no-hem-ids", + "revoke-consent", +]; + +/// The endpoint that reports a consent revocation. +const REVOKE_ENDPOINT: &str = "revoke-consent"; + +/// Lockr's token key names and the sources they carry. Only confirmed +/// mappings are listed. +const KEY_SOURCES: &[(&str, &str)] = &[ + ("id5id", "id5-sync.com"), + ("_lr_env", "liveramp.com"), + ("__uid2_advertising_token", "uidapi.com"), +]; + +impl IdentityProviderConfig for LockrConfig { + fn build_descriptor(&self) -> Result> { + let mut captures = Captures::new(); + for endpoint in CAPTURED_ENDPOINTS { + captures = captures.with_path(Method::POST, &format!("{LOCKR_API}/{endpoint}")); + } + let mut builder = + ProviderDescriptor::builder(LOCKR_INTEGRATION_ID.parse()?).with_captures(captures); + for (_, domain) in KEY_SOURCES { + builder = builder.with_source(domain.parse()?); + } + builder + .with_provenance( + &"id5-sync.com".parse()?, + EidProvenance::new() + .with_inserter("Lockr-For-Publishers") + .with_mm(2), + ) + .build() + } +} + +/// The Lockr capture provider. +pub struct LockrIdentityProvider { + descriptor: ProviderDescriptor, +} + +impl LockrIdentityProvider { + /// Builds the provider from Lockr's validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the descriptor is + /// invalid. + pub fn new(config: &LockrConfig) -> Result> { + Ok(Self { + descriptor: config.build_descriptor()?, + }) + } +} + +#[derive(Deserialize)] +struct TokenReply { + #[serde(rename = "aimTokens", default)] + aim_tokens: Vec, + #[serde(default)] + ids: BTreeMap, +} + +#[derive(Deserialize)] +struct AimToken { + key_name: String, + /// Milliseconds since the epoch. + identity_expires: Option, +} + +#[derive(Deserialize)] +struct LockrId { + eid: VendorEid, +} + +impl IdentityProvider for LockrIdentityProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn capture( + &self, + input: &CaptureInput<'_>, + ) -> Result> { + if input + .path() + .rsplit('/') + .next() + .is_some_and(|endpoint| endpoint == REVOKE_ENDPOINT) + { + return Ok(Resolution::opted_out()); + } + let Ok(reply) = input.response().json::() else { + return Ok(Resolution::no_match()); + }; + let eids: Vec = reply + .ids + .iter() + .filter_map(|(key, entry)| { + let (_, domain) = KEY_SOURCES.iter().find(|(name, _)| name == key)?; + let source = input.source_named(domain)?; + if !source.matches(&entry.eid.source) { + return None; + } + let expires = reply + .aim_tokens + .iter() + .find(|token| token.key_name == *key) + .and_then(|token| token.identity_expires) + .map(UnixTime::from_millis); + if expires.is_some_and(|at| at <= input.now()) { + return None; + } + let eid = entry.eid.to_resolved(source)?; + Some(match expires { + Some(at) => eid.with_expires_at(at), + None => eid, + }) + }) + .collect(); + Ok(if eids.is_empty() { + Resolution::no_match() + } else { + Resolution::found(eids) + }) + } +} + +#[cfg(test)] +mod tests { + use http::StatusCode; + use serde_json::json; + + use super::*; + use crate::ec::identity::When; + use crate::ec::identity::test_support::{CaptureFixture, ConformanceCases, ProviderHarness}; + + fn config() -> LockrConfig { + serde_json::from_value(json!({ "app_id": "app-1", "capture_identity": true })) + .expect("should parse the Lockr config") + } + + fn provider() -> LockrIdentityProvider { + LockrIdentityProvider::new(&config()).expect("should build the provider") + } + + fn path(endpoint: &str) -> String { + format!("{LOCKR_API}/{endpoint}") + } + + #[test] + fn a_revocation_opts_out() { + let provider = provider(); + + let resolution = ProviderHarness::new(&provider) + .capture(&CaptureFixture::new(Method::POST, &path("revoke-consent"))) + .expect("should capture"); + + assert!(resolution.is_opted_out(), "should opt the visitor out"); + } + + #[test] + fn reads_complete_eids_for_known_keys() { + let provider = provider(); + let fixture = CaptureFixture::new(Method::POST, &path("generate-tokens")).with_response( + StatusCode::OK, + &json!({ + "aimTokens": [{ "key_name": "id5id", "identity_expires": 1_770_000_000_000_u64 }], + "ids": { + "id5id": { "eid": { + "source": "id5-sync.com", + "uids": [{ "id": "ID5*example", "atype": 1 }], + "inserter": "inserter.example.com" + }}, + "unknown": { "eid": { + "source": "other.example.com", + "uids": [{ "id": "ignored" }] + }} + } + }), + ); + + let resolution = ProviderHarness::new(&provider) + .capture(&fixture) + .expect("should capture"); + + let eids = resolution.eids(); + assert_eq!(eids.len(), 1, "should keep only mapped keys"); + assert_eq!( + eids[0].source().as_str(), + "id5-sync.com", + "should map id5id" + ); + assert_eq!( + eids[0].inserter(), + Some("inserter.example.com"), + "should keep the EID fields" + ); + } + + #[test] + fn skips_an_expired_token() { + let provider = provider(); + let fixture = CaptureFixture::new(Method::POST, &path("refresh-tokens")).with_response( + StatusCode::OK, + &json!({ + "aimTokens": [{ "key_name": "id5id", "identity_expires": 1_000_u64 }], + "ids": { "id5id": { "eid": { + "source": "id5-sync.com", + "uids": [{ "id": "ID5*expired" }] + }}} + }), + ); + + let resolution = ProviderHarness::new(&provider) + .capture(&fixture) + .expect("should capture"); + + assert!(resolution.is_no_match(), "should skip the expired token"); + } + + fn tokens(path_endpoint: &str, body: &serde_json::Value) -> CaptureFixture { + CaptureFixture::new(Method::POST, &path(path_endpoint)).with_response(StatusCode::OK, body) + } + + #[test] + fn captures_exactly_the_proxied_identity_endpoints() { + let descriptor = config().build_descriptor().expect("should build"); + let captures = descriptor.captures().expect("should capture"); + + for endpoint in CAPTURED_ENDPOINTS { + assert!( + captures.matches(&Method::POST, &path(endpoint)), + "should capture POST {endpoint}" + ); + assert!( + !captures.matches(&Method::GET, &path(endpoint)), + "should not capture GET {endpoint}" + ); + } + assert!( + !captures.matches(&Method::POST, &path("other")), + "should not capture another endpoint" + ); + assert!( + !captures.matches(&Method::POST, &format!("{}/x", path("revoke-consent"))), + "should not capture a longer path" + ); + assert!(descriptor.resolves().is_none(), "should never resolve"); + } + + #[test] + fn a_revocation_opts_out_whatever_the_body() { + let provider = provider(); + let fixture = tokens( + "revoke-consent", + &json!({ "ids": { "id5id": { "eid": { "source": "id5-sync.com", "uids": [{ "id": "x" }] } } } }), + ); + + let resolution = ProviderHarness::new(&provider) + .capture(&fixture) + .expect("should capture"); + + assert!(resolution.is_opted_out(), "should opt out"); + assert!( + resolution.eids().is_empty(), + "should report no ID on a revocation" + ); + } + + #[test] + fn maps_each_known_key_to_its_source() { + let provider = provider(); + let fixture = tokens( + "page-view", + &json!({ "ids": { + "id5id": { "eid": { "source": "id5-sync.com", "uids": [{ "id": "id5" }] } }, + "_lr_env": { "eid": { "source": "liveramp.com", "uids": [{ "id": "env" }] } }, + "__uid2_advertising_token": { "eid": { "source": "uidapi.com", "uids": [{ "id": "uid2" }] } } + }}), + ); + + let resolution = ProviderHarness::new(&provider) + .capture(&fixture) + .expect("should capture"); + + let mut sources: Vec<&str> = resolution + .eids() + .iter() + .map(|eid| eid.source().as_str()) + .collect(); + sources.sort_unstable(); + assert_eq!( + sources, + ["id5-sync.com", "liveramp.com", "uidapi.com"], + "should map all three" + ); + } + + #[test] + fn skips_an_entry_whose_source_does_not_match_its_key() { + let provider = provider(); + let fixture = tokens( + "sync-no-hem-ids", + &json!({ "ids": { "id5id": { "eid": { "source": "uidapi.com", "uids": [{ "id": "x" }] } } } }), + ); + + let resolution = ProviderHarness::new(&provider) + .capture(&fixture) + .expect("should capture"); + + assert!( + resolution.is_no_match(), + "should not file a UID under another key's source" + ); + } + + #[test] + fn a_token_without_an_expiry_keeps_no_expiry_and_a_future_one_keeps_it() { + let provider = provider(); + let fixture = tokens( + "generate-tokens", + &json!({ + "aimTokens": [{ "key_name": "_lr_env", "identity_expires": 9_999_999_999_000_u64 }], + "ids": { + "id5id": { "eid": { "source": "id5-sync.com", "uids": [{ "id": "id5" }] } }, + "_lr_env": { "eid": { "source": "liveramp.com", "uids": [{ "id": "env" }] } } + } + }), + ); + + let resolution = ProviderHarness::new(&provider) + .capture(&fixture) + .expect("should capture"); + + for eid in resolution.eids() { + match eid.source().as_str() { + "id5-sync.com" => assert_eq!(eid.expiry(), None, "should not invent an expiry"), + "liveramp.com" => assert_eq!( + eid.expiry(), + Some(When::At(UnixTime::from_millis(9_999_999_999_000))), + "should keep the token expiry" + ), + other => panic!("should not resolve {other}"), + } + } + assert_eq!(resolution.eids().len(), 2, "should resolve both"); + } + + #[test] + fn a_body_it_cannot_read_is_no_match_not_an_error() { + let provider = provider(); + for body in [ + json!("text"), + json!([1, 2]), + json!({ "ids": { "id5id": { "eid": 5 } } }), + ] { + let resolution = ProviderHarness::new(&provider) + .capture(&tokens("page-view", &body)) + .expect("should not fail the proxied call"); + + assert!(resolution.is_no_match(), "should be no match for {body}"); + } + } + + #[test] + fn an_entry_without_a_uid_is_skipped() { + let provider = provider(); + let fixture = tokens( + "page-view", + &json!({ "ids": { "id5id": { "eid": { "source": "id5-sync.com", "uids": [{ "id": "" }] } } } }), + ); + + let resolution = ProviderHarness::new(&provider) + .capture(&fixture) + .expect("should capture"); + + assert!(resolution.is_no_match(), "should skip an empty UID"); + } + + #[test] + fn declares_lockr_provenance_for_id5() { + let descriptor = config().build_descriptor().expect("should build"); + + let provenance = descriptor + .provenance(&"id5-sync.com".parse().expect("should parse")) + .expect("should declare ID5 provenance"); + + assert_eq!( + provenance.inserter(), + Some("Lockr-For-Publishers"), + "should name Lockr" + ); + assert_eq!(provenance.mm(), Some(2), "should mark the match method"); + } + + #[test] + fn conforms() { + ProviderHarness::new(&provider()).check_conformance( + &ConformanceCases::default() + .with_capture_case(CaptureFixture::new(Method::POST, &path("revoke-consent"))), + ); + } +} diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index ae193625d6..307074b0f2 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -13,13 +13,18 @@ use crate::settings::Settings; pub mod adserver_mock; pub mod aps; +pub mod connectid; pub mod datadome; pub mod didomi; pub mod google_tag_manager; pub mod gpt; pub mod gpt_diagnostics; +pub mod id5; pub mod js_asset_proxy; +pub mod liveintent; +pub mod liveramp; pub mod lockr; +mod lockr_identity; pub mod nextjs; pub mod osano; pub mod permutive; @@ -27,6 +32,9 @@ pub mod prebid; mod registry; pub mod sourcepoint; pub mod testlight; +pub mod uid2; +mod vendor_eid; +mod vendor_request; pub use registry::{ AttributeRewriteAction, AttributeRewriteOutcome, HeaderMutation, HeaderMutationMode, @@ -313,6 +321,26 @@ pub(crate) fn builders() -> &'static [IntegrationBuilder] { id: "lockr", build: lockr::register, }, + IntegrationBuilder { + id: id5::ID5_INTEGRATION_ID, + build: id5::register, + }, + IntegrationBuilder { + id: liveintent::LIVEINTENT_INTEGRATION_ID, + build: liveintent::register, + }, + IntegrationBuilder { + id: liveramp::LIVERAMP_INTEGRATION_ID, + build: liveramp::register, + }, + IntegrationBuilder { + id: uid2::UID2_INTEGRATION_ID, + build: uid2::register, + }, + IntegrationBuilder { + id: connectid::CONNECTID_INTEGRATION_ID, + build: connectid::register, + }, IntegrationBuilder { id: "didomi", build: didomi::register, diff --git a/crates/trusted-server-core/src/integrations/uid2.rs b/crates/trusted-server-core/src/integrations/uid2.rs new file mode 100644 index 0000000000..b0a9a74134 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/uid2.rs @@ -0,0 +1,963 @@ +//! `UID2` identity provider: generates and refreshes `UID2` tokens server +//! side. +//! +//! A first token is generated from the hashed email in the `ts-hem` cookie, +//! sent in an AES-256-GCM envelope keyed by the operator's client secret. +//! The refresh token and the key that decrypts its response are kept as +//! private state, so later visits refresh without the hashed email. + +use aes_gcm::aead::{Aead as _, KeyInit as _}; +use aes_gcm::{Aes256Gcm, Nonce}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use error_stack::Report; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use url::Url; +use validator::Validate; + +use crate::ec::identity::{ + ConsentPolicy, IdentityProvider, IdentityProviderConfig, IdentityProviderError, Input, + ProviderDescriptor, ProviderRequest, ProviderResponse, Resolution, ResolveInput, ResolveKind, + ResolvedEid, ResolvedUid, Resolves, UnixTime, +}; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistration; +use crate::redacted::Redacted; +use crate::settings::{IntegrationConfig, Settings}; + +/// Integration ID of the `UID2` provider. +pub(crate) const UID2_INTEGRATION_ID: &str = "uid2"; + +/// The envelope version byte. +const ENVELOPE_VERSION: u8 = 1; +const IV_BYTES: usize = 12; +const TAG_BYTES: usize = 16; +const TIMESTAMP_BYTES: usize = 8; +const NONCE_BYTES: usize = 8; + +/// Config of the `uid2` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct Uid2Config { + /// Enables the provider. + #[serde(default)] + pub enabled: bool, + /// The operator base URL, for example `https://prod.uidapi.com`. + #[serde(default = "default_base_url")] + #[validate(url)] + pub base_url: String, + /// The API key. A secret-store key name at rest, the resolved value at + /// runtime. + pub api_key: Redacted, + /// The base64 client secret that keys request envelopes. A secret-store + /// key name at rest, the resolved value at runtime. + pub client_secret: Redacted, + /// The IAB Global Vendor List ID whose consent `UID2` calls require where + /// TCF applies, if any. + pub tcf_vendor_id: Option, +} + +fn default_base_url() -> String { + "https://prod.uidapi.com".to_owned() +} + +impl IntegrationConfig for Uid2Config { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl IdentityProviderConfig for Uid2Config { + fn build_descriptor(&self) -> Result> { + // UID2 has no GVL ID, so it keeps Purpose 4 (personalised ads) as + // its declared purpose. An operator whose UID2 operator is a TCF + // vendor names its ID; otherwise purpose consent alone applies. + let consent = ConsentPolicy::strict().with_tcf_purposes([4]); + let consent = match self.tcf_vendor_id { + Some(vendor) => consent.with_tcf_vendor(vendor), + None => consent.without_tcf_vendor(), + }; + ProviderDescriptor::builder(UID2_INTEGRATION_ID.parse()?) + .with_source("uidapi.com".parse()?) + .with_expiring_ids() + .with_resolves( + Resolves::new() + .with_endpoint(&self.base_url) + .with_needs_for(ResolveKind::Initial, [Input::HashedEmail]) + .with_private_state(), + ) + .with_consent_policy(consent) + .build() + } +} + +/// The `UID2` provider. +pub struct Uid2Provider { + descriptor: ProviderDescriptor, + api_key: Redacted, + client_key: [u8; 32], +} + +impl Uid2Provider { + /// Builds the provider from its config with resolved secrets. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the client secret is + /// not 32 base64 bytes, or the config cannot describe a valid provider. + pub fn new(config: &Uid2Config) -> Result> { + let client_key = STANDARD + .decode(config.client_secret.expose().trim()) + .ok() + .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok()) + .ok_or_else(|| { + IdentityProviderError::config("`client_secret` is not 32 base64-encoded bytes") + })?; + Ok(Self { + descriptor: config.build_descriptor()?, + api_key: config.api_key.clone(), + client_key, + }) + } +} + +/// The refresh anchor kept between calls. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct Uid2State { + refresh_token: String, + refresh_response_key: String, + /// Milliseconds since the epoch. + refresh_expires: u64, +} + +/// The nonce a generate call sent, which its response must echo. +#[derive(Serialize, Deserialize)] +struct GenerateCall { + request_nonce: [u8; NONCE_BYTES], +} + +#[derive(Deserialize)] +struct Uid2Reply { + status: String, + body: Option, +} + +#[derive(Deserialize)] +struct Uid2Identity { + advertising_token: String, + refresh_token: String, + refresh_response_key: String, + /// Milliseconds since the epoch, like the two below. + identity_expires: u64, + refresh_from: u64, + refresh_expires: u64, +} + +/// `path` under `base`, keeping a `base` path that has no trailing slash. +fn join(base: &Url, path: &str) -> Result> { + let mut base = base.clone(); + if !base.path().ends_with('/') { + let directory = format!("{}/", base.path()); + base.set_path(&directory); + } + base.join(path) + .map_err(|_| IdentityProviderError::config("`base_url` cannot take a path")) +} + +/// AES-256-GCM of `plaintext` under `key`, as `ciphertext || tag`. +fn seal( + key: &[u8; 32], + iv: &[u8; IV_BYTES], + plaintext: &[u8], +) -> Result, Report> { + Aes256Gcm::new(key.into()) + .encrypt(Nonce::from_slice(iv), plaintext) + .map_err(|_| IdentityProviderError::config("envelope encryption failed")) +} + +/// Opens base64 `IV || ciphertext || tag` under `key`. +fn open(key: &[u8], body: &str) -> Result, Report> { + let bytes = STANDARD + .decode(body.trim()) + .map_err(|_| IdentityProviderError::malformed())?; + if bytes.len() < IV_BYTES + TAG_BYTES { + return Err(IdentityProviderError::malformed()); + } + let cipher = Aes256Gcm::new_from_slice(key).map_err(|_| IdentityProviderError::malformed())?; + let (iv, sealed) = bytes.split_at(IV_BYTES); + cipher + .decrypt(Nonce::from_slice(iv), sealed) + .map_err(|_| IdentityProviderError::malformed()) +} + +/// The token-generate envelope: base64 of the version byte, the IV, and the +/// sealed timestamp, nonce and JSON payload. +fn encrypt_envelope( + key: &[u8; 32], + iv: &[u8; IV_BYTES], + now: UnixTime, + nonce: &[u8; NONCE_BYTES], + payload: &serde_json::Value, +) -> Result> { + let mut plaintext = Vec::with_capacity(TIMESTAMP_BYTES + NONCE_BYTES + 64); + plaintext.extend_from_slice(&now.as_secs().saturating_mul(1000).to_be_bytes()); + plaintext.extend_from_slice(nonce); + plaintext.extend_from_slice(payload.to_string().as_bytes()); + let sealed = seal(key, iv, &plaintext)?; + let mut envelope = Vec::with_capacity(1 + IV_BYTES + sealed.len()); + envelope.push(ENVELOPE_VERSION); + envelope.extend_from_slice(iv); + envelope.extend_from_slice(&sealed); + Ok(STANDARD.encode(envelope)) +} + +/// Opens a generate response and checks that it echoes `nonce`. +fn decrypt_generate( + key: &[u8; 32], + body: &str, + nonce: &[u8; NONCE_BYTES], +) -> Result> { + let plaintext = open(key, body)?; + let payload = plaintext + .get(TIMESTAMP_BYTES..) + .and_then(|rest| rest.strip_prefix(nonce.as_slice())) + .ok_or_else(IdentityProviderError::malformed)?; + serde_json::from_slice(payload).map_err(|_| IdentityProviderError::malformed()) +} + +/// Opens a refresh response with the stored base64 response key. +fn decrypt_refresh(key: &str, body: &str) -> Result> { + let key = STANDARD + .decode(key) + .map_err(|_| IdentityProviderError::malformed())?; + let plaintext = open(&key, body)?; + serde_json::from_slice(&plaintext).map_err(|_| IdentityProviderError::malformed()) +} + +impl IdentityProvider for Uid2Provider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let base = input.endpoint()?; + if input.is_refresh(input.primary_source()) { + let Some(state) = input.state::()? else { + return Ok(None); + }; + if UnixTime::from_millis(state.refresh_expires) <= input.now() { + return Ok(None); + } + let url = join(base, "v2/token/refresh")?; + return Ok(Some( + ProviderRequest::post_text(&url, &state.refresh_token).with_bearer(&self.api_key), + )); + } + let url = join(base, "v2/token/generate")?; + let request_nonce = input.random_bytes::(); + let envelope = encrypt_envelope( + &self.client_key, + &input.random_bytes::(), + input.now(), + &request_nonce, + &json!({ "email_hash": input.hashed_email()?.to_base64(), "optout_check": 1 }), + )?; + Ok(Some( + ProviderRequest::post_text(&url, &envelope) + .with_bearer(&self.api_key) + .try_with_call_context(&GenerateCall { request_nonce })?, + )) + } + + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let body = response.success_text()?; + let reply = match response.call_context::()? { + Some(call) => decrypt_generate(&self.client_key, body, &call.request_nonce)?, + None => { + let state = input + .state::()? + .ok_or_else(|| IdentityProviderError::config("refresh without state"))?; + decrypt_refresh(&state.refresh_response_key, body)? + } + }; + match reply.status.as_str() { + "optout" => Ok(Resolution::opted_out().clearing_state()), + "expired_token" | "invalid_token" => { + Ok(Resolution::invalidate(input.primary_source()).clearing_state()) + } + "success" => { + let identity = reply.body.ok_or_else(IdentityProviderError::malformed)?; + let uid = ResolvedUid::new(identity.advertising_token).with_atype(3); + let eid = ResolvedEid::new(input.primary_source(), uid) + .with_expires_at(UnixTime::from_millis(identity.identity_expires)) + .with_refresh_at(UnixTime::from_millis(identity.refresh_from)) + .with_refresh_until(UnixTime::from_millis(identity.refresh_expires)); + Ok(Resolution::found([eid]).replacing_state(&Uid2State { + refresh_token: identity.refresh_token, + refresh_response_key: identity.refresh_response_key, + refresh_expires: identity.refresh_expires, + })) + } + _ => Err(IdentityProviderError::malformed()), + } + } +} + +/// Registers the `UID2` provider when `[integrations.uid2]` is enabled. +/// +/// # Errors +/// +/// Returns an error when the config does not parse, the client secret is +/// invalid, or the config describes an invalid provider. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.integration_config::(UID2_INTEGRATION_ID)? else { + return Ok(None); + }; + log::info!("Registering UID2 identity provider"); + Ok(Some( + IntegrationRegistration::builder(UID2_INTEGRATION_ID) + .try_with_identity_provider(Uid2Provider::new(&config))? + .without_js() + .build(), + )) +} + +#[cfg(test)] +mod tests { + use http::StatusCode; + + use super::*; + use crate::ec::identity::test_support::{ + ConformanceCases, HarnessSetup, ProviderHarness, check_descriptor_ignores_secrets, + }; + use crate::ec::identity::{HashedEmail, SourceDomain, StateChange, StoredId, When}; + + const CLIENT_KEY: [u8; 32] = [7; 32]; + const RESPONSE_KEY: [u8; 32] = [9; 32]; + + fn config(api_key: &str, client_secret: &str) -> Uid2Config { + Uid2Config { + enabled: true, + base_url: "https://uid2.example.com".to_owned(), + api_key: Redacted::new(api_key.to_owned()), + client_secret: Redacted::new(client_secret.to_owned()), + tcf_vendor_id: None, + } + } + + fn provider() -> Uid2Provider { + Uid2Provider::new(&config("api-key", &STANDARD.encode(CLIENT_KEY))) + .expect("should build the provider") + } + + fn sealed_response(key: &[u8; 32], plaintext: &[u8]) -> String { + let iv = [3u8; IV_BYTES]; + let mut bytes = iv.to_vec(); + bytes.extend(seal(key, &iv, plaintext).expect("should seal")); + STANDARD.encode(bytes) + } + + fn identity() -> serde_json::Value { + json!({ + "status": "success", + "body": { + "advertising_token": "advertising-token", + "refresh_token": "refresh-token", + "refresh_response_key": STANDARD.encode(RESPONSE_KEY), + "identity_expires": 1_760_003_600_000_u64, + "refresh_from": 1_760_001_800_000_u64, + "refresh_expires": 1_762_592_000_000_u64 + } + }) + } + + #[test] + fn generates_a_token_from_the_hashed_email() { + let provider = provider(); + let mut harness = + ProviderHarness::new(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + assert_eq!( + request.url().as_str(), + "https://uid2.example.com/v2/token/generate", + "should call token/generate" + ); + let envelope = STANDARD + .decode(request.body()) + .expect("should send a base64 envelope"); + assert_eq!( + envelope[0], ENVELOPE_VERSION, + "should lead with the version" + ); + let plaintext = Aes256Gcm::new((&CLIENT_KEY).into()) + .decrypt( + Nonce::from_slice(&envelope[1..=IV_BYTES]), + &envelope[1 + IV_BYTES..], + ) + .expect("should open with the client secret"); + let nonce: [u8; NONCE_BYTES] = plaintext[TIMESTAMP_BYTES..TIMESTAMP_BYTES + NONCE_BYTES] + .try_into() + .expect("should hold a nonce"); + let payload: serde_json::Value = + serde_json::from_slice(&plaintext[TIMESTAMP_BYTES + NONCE_BYTES..]) + .expect("should hold JSON"); + assert_eq!( + payload["email_hash"], + STANDARD.encode([0xab; 32]), + "should send the base64 hash" + ); + + let mut reply = vec![0u8; TIMESTAMP_BYTES]; + reply.extend_from_slice(&nonce); + reply.extend(identity().to_string().into_bytes()); + let resolution = harness + .respond_raw( + StatusCode::OK, + &[], + sealed_response(&CLIENT_KEY, &reply).as_bytes(), + ) + .expect("should decrypt the response"); + + assert_eq!( + resolution.eids()[0].uids()[0].id(), + "advertising-token", + "should store the advertising token" + ); + let state: Uid2State = harness.state_of(&resolution).expect("should keep state"); + assert_eq!( + state.refresh_token, "refresh-token", + "should keep the refresh token" + ); + } + + #[test] + fn rejects_a_generate_response_with_another_nonce() { + let provider = provider(); + let mut harness = + ProviderHarness::new(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let mut reply = vec![0u8; TIMESTAMP_BYTES + NONCE_BYTES]; + reply.extend(identity().to_string().into_bytes()); + let error = harness + .respond_raw( + StatusCode::OK, + &[], + sealed_response(&CLIENT_KEY, &reply).as_bytes(), + ) + .expect_err("should refuse a mismatched nonce"); + + assert!( + matches!(error.current_context(), IdentityProviderError::Malformed), + "should be malformed" + ); + } + + #[test] + fn refreshes_with_the_stored_state() { + let provider = provider(); + let source: SourceDomain = "uidapi.com".parse().expect("should parse the source"); + let mut harness = ProviderHarness::new(&provider).with_setup( + HarnessSetup::default() + .with_stored( + StoredId::for_test(&source, "old-token").expiring_at(UnixTime::from_secs(1)), + ) + .with_state(&Uid2State { + refresh_token: "stored-refresh".to_owned(), + refresh_response_key: STANDARD.encode(RESPONSE_KEY), + refresh_expires: 1_770_000_000_000, + }), + ); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + assert_eq!( + request.body(), + b"stored-refresh", + "should send the refresh token" + ); + + let resolution = harness + .respond_raw( + StatusCode::OK, + &[], + sealed_response( + &RESPONSE_KEY, + json!({ "status": "optout" }).to_string().as_bytes(), + ) + .as_bytes(), + ) + .expect("should decrypt the refresh response"); + assert!(resolution.is_opted_out(), "should report the opt-out"); + } + + fn source() -> SourceDomain { + "uidapi.com".parse().expect("should parse the source") + } + + fn state(refresh_expires: u64) -> Uid2State { + Uid2State { + refresh_token: "stored-refresh".to_owned(), + refresh_response_key: STANDARD.encode(RESPONSE_KEY), + refresh_expires, + } + } + + fn refreshing(state: &Uid2State) -> HarnessSetup { + HarnessSetup::default() + .with_stored( + StoredId::for_test(&source(), "old-token").expiring_at(UnixTime::from_secs(1)), + ) + .with_state(state) + } + + fn refresh_reply(reply: &serde_json::Value) -> String { + sealed_response(&RESPONSE_KEY, reply.to_string().as_bytes()) + } + + fn generate_reply(request: &ProviderRequest, reply: &serde_json::Value) -> String { + let envelope = STANDARD.decode(request.body()).expect("should be base64"); + let plaintext = Aes256Gcm::new((&CLIENT_KEY).into()) + .decrypt( + Nonce::from_slice(&envelope[1..=IV_BYTES]), + &envelope[1 + IV_BYTES..], + ) + .expect("should open"); + let mut sealed = vec![0u8; TIMESTAMP_BYTES]; + sealed.extend_from_slice(&plaintext[TIMESTAMP_BYTES..TIMESTAMP_BYTES + NONCE_BYTES]); + sealed.extend(reply.to_string().into_bytes()); + sealed_response(&CLIENT_KEY, &sealed) + } + + fn with_email(provider: &Uid2Provider) -> ProviderHarness<'_> { + ProviderHarness::new(provider).with_hashed_email(HashedEmail::for_test([0xab; 32])) + } + + fn requested(harness: &mut ProviderHarness<'_>) -> ProviderRequest { + harness + .request() + .expect("should build a request") + .expect("should not skip") + } + + #[test] + fn skips_a_first_token_without_a_hashed_email() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + + assert!( + harness.request().expect("should not fail").is_none(), + "should not call UID2 without a HEM" + ); + } + + #[test] + fn generate_sends_the_api_key_as_a_bearer_and_the_time_in_millis() { + let provider = provider(); + let mut harness = with_email(&provider).with_now(UnixTime::from_secs(1_760_000_000)); + + let request = requested(&mut harness); + + assert_eq!( + request + .headers() + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some("Bearer api-key"), + "should authenticate with the API key" + ); + let envelope = STANDARD.decode(request.body()).expect("should be base64"); + let plaintext = Aes256Gcm::new((&CLIENT_KEY).into()) + .decrypt( + Nonce::from_slice(&envelope[1..=IV_BYTES]), + &envelope[1 + IV_BYTES..], + ) + .expect("should open"); + let millis = u64::from_be_bytes( + plaintext[..TIMESTAMP_BYTES] + .try_into() + .expect("should hold a timestamp"), + ); + assert_eq!( + millis, 1_760_000_000_000, + "should stamp the request in millis" + ); + let payload: serde_json::Value = + serde_json::from_slice(&plaintext[TIMESTAMP_BYTES + NONCE_BYTES..]).expect("JSON"); + assert_eq!( + payload["optout_check"], 1, + "should ask UID2 to check opt-outs" + ); + } + + #[test] + fn each_generate_uses_a_fresh_iv_and_nonce() { + let provider = provider(); + let mut first = with_email(&provider).with_random_seed(1); + let mut second = with_email(&provider).with_random_seed(2); + + let a = STANDARD + .decode(requested(&mut first).body()) + .expect("base64"); + let b = STANDARD + .decode(requested(&mut second).body()) + .expect("base64"); + + assert_ne!(a[1..=IV_BYTES], b[1..=IV_BYTES], "should not reuse an IV"); + assert_ne!(a, b, "should not repeat an envelope"); + } + + #[test] + fn generate_success_sets_expiry_refresh_and_limit_from_the_identity() { + let provider = provider(); + let mut harness = with_email(&provider); + let request = requested(&mut harness); + + let resolution = harness + .respond_raw( + StatusCode::OK, + &[], + generate_reply(&request, &identity()).as_bytes(), + ) + .expect("should decrypt"); + + let eid = &resolution.eids()[0]; + assert_eq!(eid.source(), &source(), "should write uidapi.com"); + assert_eq!(eid.uids()[0].atype(), Some(3), "should be atype 3"); + assert_eq!( + eid.expiry(), + Some(When::At(UnixTime::from_millis(1_760_003_600_000))), + "should expire at identity_expires" + ); + assert_eq!( + eid.refresh(), + Some(When::At(UnixTime::from_millis(1_760_001_800_000))), + "should refresh from refresh_from" + ); + assert_eq!( + eid.refresh_until(), + Some(UnixTime::from_millis(1_762_592_000_000)), + "should stop refreshing at refresh_expires" + ); + } + + #[test] + fn generate_opt_out_reports_the_opt_out_and_clears_state() { + let provider = provider(); + let mut harness = with_email(&provider); + let request = requested(&mut harness); + + let resolution = harness + .respond_raw( + StatusCode::OK, + &[], + generate_reply(&request, &json!({ "status": "optout" })).as_bytes(), + ) + .expect("should decrypt"); + + assert!(resolution.is_opted_out(), "should be opted out"); + assert!( + matches!(resolution.state_change(), StateChange::Clear), + "should clear state" + ); + assert!(resolution.eids().is_empty(), "should return no token"); + } + + #[test] + fn refresh_success_replaces_the_state_with_the_new_refresh_token() { + let provider = provider(); + let stored = state(1_770_000_000_000); + let mut harness = ProviderHarness::new(&provider).with_setup(refreshing(&stored)); + let request = requested(&mut harness); + assert_eq!( + request.url().as_str(), + "https://uid2.example.com/v2/token/refresh", + "should call token/refresh" + ); + assert!( + request.headers().get(http::header::AUTHORIZATION).is_some(), + "should authenticate the refresh" + ); + + let mut reply = identity(); + reply["body"]["refresh_token"] = json!("next-refresh"); + let resolution = harness + .respond_raw(StatusCode::OK, &[], refresh_reply(&reply).as_bytes()) + .expect("should decrypt with the response key"); + + let next: Uid2State = harness.state_of(&resolution).expect("should replace state"); + assert_eq!( + next.refresh_token, "next-refresh", + "should rotate the refresh token" + ); + assert_eq!( + resolution.eids()[0].uids()[0].id(), + "advertising-token", + "should keep the token" + ); + } + + #[test] + fn a_refused_refresh_token_invalidates_and_clears_state() { + for status in ["expired_token", "invalid_token"] { + let provider = provider(); + let stored = state(1_770_000_000_000); + let mut harness = ProviderHarness::new(&provider).with_setup(refreshing(&stored)); + requested(&mut harness); + + let resolution = harness + .respond_raw( + StatusCode::OK, + &[], + refresh_reply(&json!({ "status": status })).as_bytes(), + ) + .expect("should decrypt"); + + assert_eq!( + resolution.invalidated(), + [source()], + "should drop the token for {status}" + ); + assert!( + matches!(resolution.state_change(), StateChange::Clear), + "should clear state" + ); + } + } + + #[test] + fn skips_a_refresh_whose_refresh_token_has_expired() { + let provider = provider(); + let stored = state(1_000); + let mut harness = ProviderHarness::new(&provider) + .with_setup(refreshing(&stored).with_now(UnixTime::from_secs(1_760_000_000))); + + assert!( + harness.request().expect("should not fail").is_none(), + "should not send a refresh token UID2 would refuse" + ); + } + + #[test] + fn skips_a_refresh_without_state() { + let provider = provider(); + let mut harness = + ProviderHarness::new(&provider).with_setup(HarnessSetup::default().with_stored( + StoredId::for_test(&source(), "old-token").expiring_at(UnixTime::from_secs(1)), + )); + + assert!( + harness.request().expect("should not fail").is_none(), + "should not refresh without a refresh token" + ); + } + + #[test] + fn an_unknown_status_or_a_success_without_body_is_malformed() { + for reply in [ + json!({ "status": "client_error" }), + json!({ "status": "success" }), + ] { + let provider = provider(); + let stored = state(1_770_000_000_000); + let mut harness = ProviderHarness::new(&provider).with_setup(refreshing(&stored)); + requested(&mut harness); + + let error = harness + .respond_raw(StatusCode::OK, &[], refresh_reply(&reply).as_bytes()) + .expect_err("should fail"); + + assert!( + matches!(error.current_context(), IdentityProviderError::Malformed), + "should be malformed for {reply}: {error:?}" + ); + } + } + + #[test] + fn a_tampered_short_or_non_base64_body_is_malformed() { + let mut tampered = STANDARD + .decode(refresh_reply(&identity())) + .expect("should be base64"); + let last = tampered.len() - 1; + tampered[last] ^= 1; + let bodies = [ + STANDARD.encode(tampered), + STANDARD.encode([0u8; IV_BYTES + TAG_BYTES - 1]), + "%%%".to_owned(), + sealed_response(&CLIENT_KEY, identity().to_string().as_bytes()), + ]; + for body in bodies { + let provider = provider(); + let stored = state(1_770_000_000_000); + let mut harness = ProviderHarness::new(&provider).with_setup(refreshing(&stored)); + requested(&mut harness); + + let error = harness + .respond_raw(StatusCode::OK, &[], body.as_bytes()) + .expect_err("should refuse the body"); + + assert!( + matches!(error.current_context(), IdentityProviderError::Malformed), + "should be malformed: {error:?}" + ); + } + } + + #[test] + fn a_non_success_status_is_rejected_before_decryption() { + let provider = provider(); + let mut harness = with_email(&provider); + requested(&mut harness); + + let error = harness + .respond( + StatusCode::UNAUTHORIZED, + &json!({ "status": "unauthorized" }), + ) + .expect_err("should reject"); + + assert!( + matches!(error.current_context(), IdentityProviderError::Rejected { status } if *status == StatusCode::UNAUTHORIZED), + "should be rejected: {error:?}" + ); + } + + #[test] + fn rejects_a_client_secret_of_the_wrong_length_and_accepts_surrounding_whitespace() { + assert!( + Uid2Provider::new(&config("api-key", &STANDARD.encode([1u8; 16]))).is_err(), + "should refuse a 16-byte key" + ); + assert!( + Uid2Provider::new(&config( + "api-key", + &format!(" {}\n", STANDARD.encode(CLIENT_KEY)) + )) + .is_ok(), + "should trim the secret" + ); + } + + #[test] + fn the_config_debug_hides_both_secrets() { + let config = config("secret-api-key", "secret-client-secret"); + + let debug = format!("{config:?}"); + + assert!( + !debug.contains("secret-api-key"), + "should hide the API key: {debug}" + ); + assert!( + !debug.contains("secret-client-secret"), + "should hide the secret: {debug}" + ); + } + + #[test] + fn a_tcf_vendor_id_adds_vendor_consent_only_when_set() { + let without = config("k", &STANDARD.encode(CLIENT_KEY)) + .build_descriptor() + .expect("should build"); + assert_eq!( + without.consent_policy().tcf_vendor(), + None, + "should name no vendor by default" + ); + + let with = Uid2Config { + tcf_vendor_id: Some(21), + ..config("k", &STANDARD.encode(CLIENT_KEY)) + } + .build_descriptor() + .expect("should build"); + assert_eq!( + with.consent_policy().tcf_vendor(), + Some(21), + "should name the vendor" + ); + assert!(with.is_expiring(&source()), "tokens should expire"); + } + + #[test] + fn conforms() { + let stored = state(1_770_000_000_000); + ProviderHarness::new(&provider()).check_conformance_with_state::( + &ConformanceCases::default().with_resolve_case_raw( + refreshing(&stored), + StatusCode::OK, + &[], + refresh_reply(&identity()).as_bytes(), + ), + ); + } + + #[test] + fn keeps_a_base_url_path_with_or_without_a_trailing_slash() { + for (base, expected) in [ + ( + "https://uid2.example.com", + "https://uid2.example.com/v2/token/generate", + ), + ( + "https://uid2.example.com/", + "https://uid2.example.com/v2/token/generate", + ), + ( + "https://uid2.example.com/operator", + "https://uid2.example.com/operator/v2/token/generate", + ), + ( + "https://uid2.example.com/operator/", + "https://uid2.example.com/operator/v2/token/generate", + ), + ] { + let provider = Uid2Provider::new(&Uid2Config { + base_url: base.to_owned(), + ..config("api-key", &STANDARD.encode(CLIENT_KEY)) + }) + .expect("should build the provider"); + let mut harness = with_email(&provider); + + let request = requested(&mut harness); + + assert_eq!( + request.url().as_str(), + expected, + "should keep the path of {base}" + ); + } + } + + #[test] + fn rejects_a_client_secret_that_is_not_a_key() { + assert!( + Uid2Provider::new(&config("api-key", "not-base64-key")).is_err(), + "should refuse an invalid client secret" + ); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets( + &config("api-key-name", "client-secret-name"), + &config("resolved-api-key", &STANDARD.encode(CLIENT_KEY)), + ); + } +} diff --git a/crates/trusted-server-core/src/integrations/vendor_eid.rs b/crates/trusted-server-core/src/integrations/vendor_eid.rs new file mode 100644 index 0000000000..8ea67b18f6 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/vendor_eid.rs @@ -0,0 +1,136 @@ +//! Permissive parsing of complete EIDs that identity vendors return. +//! +//! [`crate::openrtb::Eid`] denies unknown fields, so it suits Trusted +//! Server's own output, not a vendor's input: one new vendor field would fail +//! the whole response. + +use serde::Deserialize; + +use crate::ec::identity::{ResolvedEid, ResolvedUid, SourceDomain}; + +/// One EID as a vendor returns it. +#[derive(Debug, Deserialize)] +pub(crate) struct VendorEid { + pub source: String, + #[serde(default)] + pub uids: Vec, + pub inserter: Option, + pub matcher: Option, + pub mm: Option, + pub ext: Option, +} + +/// One UID of a [`VendorEid`]. +#[derive(Debug, Deserialize)] +pub(crate) struct VendorUid { + pub id: String, + pub atype: Option, + pub ext: Option, +} + +impl VendorEid { + /// The EID under `source`, with every UID and EID-level field the vendor + /// sent. `None` when it holds no UID with an ID. + pub(crate) fn to_resolved(&self, source: &SourceDomain) -> Option { + let mut uids = self + .uids + .iter() + .filter(|uid| !uid.id.is_empty()) + .map(VendorUid::to_resolved); + let mut eid = ResolvedEid::new(source, uids.next()?); + for uid in uids { + eid = eid.with_uid(uid); + } + if let Some(inserter) = &self.inserter { + eid = eid.with_inserter(inserter); + } + if let Some(matcher) = &self.matcher { + eid = eid.with_matcher(matcher); + } + if let Some(mm) = self.mm { + eid = eid.with_mm(mm); + } + if let Some(ext) = &self.ext { + eid = eid.with_ext(ext.clone()); + } + Some(eid) + } +} + +impl VendorUid { + fn to_resolved(&self) -> ResolvedUid { + let mut uid = ResolvedUid::new(self.id.clone()); + if let Some(atype) = self.atype { + uid = uid.with_atype(atype); + } + if let Some(ext) = &self.ext { + uid = uid.with_ext(ext.clone()); + } + uid + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn keeps_every_uid_and_eid_field() { + let eid: VendorEid = serde_json::from_value(json!({ + "source": "ids.example.com", + "uids": [ + { "id": "first", "atype": 1, "ext": { "linkType": 2 } }, + { "id": "" }, + { "id": "second", "future": true } + ], + "inserter": "inserter.example.com", + "matcher": "matcher.example.com", + "mm": 2, + "ext": { "segments": ["s1"] }, + "future": "ignored" + })) + .expect("should parse a vendor EID with unknown fields"); + let source: SourceDomain = "ids.example.com".parse().expect("should parse the source"); + + let resolved = eid.to_resolved(&source).expect("should resolve"); + + let ids: Vec<&str> = resolved.uids().iter().map(ResolvedUid::id).collect(); + assert_eq!( + ids, + ["first", "second"], + "should keep non-empty UIDs in order" + ); + assert_eq!(resolved.uids()[0].atype(), Some(1), "should keep the atype"); + assert_eq!( + resolved.inserter(), + Some("inserter.example.com"), + "should keep inserter" + ); + assert_eq!( + resolved.matcher(), + Some("matcher.example.com"), + "should keep matcher" + ); + assert_eq!(resolved.mm(), Some(2), "should keep mm"); + assert_eq!( + resolved.ext(), + Some(&json!({ "segments": ["s1"] })), + "should keep the EID ext" + ); + } + + #[test] + fn an_eid_without_ids_resolves_to_none() { + let eid: VendorEid = + serde_json::from_value(json!({ "source": "ids.example.com", "uids": [{ "id": "" }] })) + .expect("should parse"); + let source: SourceDomain = "ids.example.com".parse().expect("should parse the source"); + + assert!( + eid.to_resolved(&source).is_none(), + "should skip an EID with no ID" + ); + } +} diff --git a/crates/trusted-server-core/src/integrations/vendor_request.rs b/crates/trusted-server-core/src/integrations/vendor_request.rs new file mode 100644 index 0000000000..d3d498be10 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/vendor_request.rs @@ -0,0 +1,67 @@ +//! Helpers shared by the identity vendors' requests. + +use std::net::IpAddr; + +/// Whether `ip` can identify a visitor to a vendor: not loopback, +/// unspecified, private, link-local or carrier-grade NAT. +/// +/// A vendor that matches on the client IP answers no match for such an +/// address, as it does for a request from a local dev server, so the IP is +/// left out instead. +pub(crate) fn is_routable_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + let [first, second, ..] = v4.octets(); + let carrier_grade_nat = first == 100 && (second & 0xc0) == 64; + !(v4.is_loopback() + || v4.is_unspecified() + || v4.is_private() + || v4.is_link_local() + || v4.is_broadcast() + || carrier_grade_nat) + } + IpAddr::V6(v6) => { + if let Some(v4) = v6.to_ipv4_mapped() { + return is_routable_ip(IpAddr::V4(v4)); + } + let first = v6.segments()[0]; + let unique_local = (first & 0xfe00) == 0xfc00; + let link_local = (first & 0xffc0) == 0xfe80; + !(v6.is_loopback() || v6.is_unspecified() || unique_local || link_local) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_public_addresses_and_drops_local_ones() { + for ip in ["192.0.2.10", "198.51.100.7", "2001:db8::1"] { + assert!( + is_routable_ip(ip.parse().expect("should parse")), + "should keep {ip}" + ); + } + for ip in [ + "127.0.0.1", + "0.0.0.0", + "10.1.2.3", + "172.16.0.1", + "192.168.1.1", + "169.254.1.1", + "100.64.0.1", + "::1", + "::", + "fd00::1", + "fe80::1", + "::ffff:127.0.0.1", + ] { + assert!( + !is_routable_ip(ip.parse().expect("should parse")), + "should drop {ip}" + ); + } + } +} diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6e919e8876..2d1101de1e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -7903,6 +7903,8 @@ mod tests { user_agent: None, dnt: false, page_url: None, + hashed_email: None, + cookies: std::collections::BTreeMap::new(), }, ); assert!( @@ -8247,6 +8249,8 @@ mod tests { user_agent: None, dnt: false, page_url: None, + hashed_email: None, + cookies: std::collections::BTreeMap::new(), }, ); ec_context.set_identity_runtime(Arc::new( @@ -8762,6 +8766,8 @@ mod tests { user_agent: None, dnt: false, page_url: None, + hashed_email: None, + cookies: std::collections::BTreeMap::new(), }, ); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); @@ -17729,6 +17735,8 @@ mod tests { user_agent: None, dnt: false, page_url: None, + hashed_email: None, + cookies: std::collections::BTreeMap::new(), }, ); context diff --git a/crates/trusted-server-js/lib/src/core/hashed_email.ts b/crates/trusted-server-js/lib/src/core/hashed_email.ts new file mode 100644 index 0000000000..af47f675fa --- /dev/null +++ b/crates/trusted-server-js/lib/src/core/hashed_email.ts @@ -0,0 +1,88 @@ +// Writes the first-party `ts-hem` cookie: the lowercase hex SHA-256 of the +// visitor's normalized email address. TS reads it per request as the identity +// providers' hashed-email input and never stores it, forwards it to the +// origin, or sends it to Prebid Server. + +const HEM_COOKIE_NAME = 'ts-hem'; + +const ATTRIBUTES = 'Path=/; Secure; SameSite=Lax'; + +// Thirty days: the page sets it again on each sign-in. +const HEM_COOKIE_MAX_AGE = 30 * 24 * 60 * 60; + +const SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/; + +const GMAIL_DOMAINS = new Set(['gmail.com', 'googlemail.com']); + +/** + * Normalizes an email address the way UID2 and most ID vendors hash it: + * trims and lowercases, and for Gmail drops dots and any `+suffix` from the + * local part. + * + * @param email the address the visitor gave + * @returns the normalized address, or `undefined` when it is not an address + */ +export function normalizeEmail(email: string): string | undefined { + const trimmed = email.trim().toLowerCase(); + const at = trimmed.indexOf('@'); + if (at <= 0 || at === trimmed.length - 1 || trimmed.includes('@', at + 1) || /\s/.test(trimmed)) { + return undefined; + } + let local = trimmed.slice(0, at); + const domain = trimmed.slice(at + 1); + if (GMAIL_DOMAINS.has(domain)) { + local = local.split('+')[0].replace(/\./g, ''); + if (local.length === 0) { + return undefined; + } + } + return `${local}@${domain}`; +} + +function writeCookie(value: string, maxAge: number): void { + try { + document.cookie = `${HEM_COOKIE_NAME}=${value}; ${ATTRIBUTES}; Max-Age=${maxAge}`; + } catch { + // Cookie access can throw in sandboxed frames; the hashed email is optional. + } +} + +/** + * Stores an already-hashed email. + * + * @param hex the SHA-256 of the normalized address, as 64 hex digits + * @returns whether the value was a digest and was stored + */ +export function setHashedEmail(hex: string): boolean { + const value = hex.trim().toLowerCase(); + if (!SHA256_HEX_PATTERN.test(value)) { + return false; + } + writeCookie(value, HEM_COOKIE_MAX_AGE); + return true; +} + +/** + * Normalizes and hashes an email address in the browser, then stores the + * digest. The address itself is never stored or sent. + * + * @param email the address the visitor gave + * @returns whether a digest was stored + */ +export async function setEmail(email: string): Promise { + const normalized = normalizeEmail(email); + const subtle = globalThis.crypto?.subtle; + if (normalized === undefined || subtle === undefined) { + return false; + } + const digest = await subtle.digest('SHA-256', new TextEncoder().encode(normalized)); + const hex = Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, '0')).join( + '' + ); + return setHashedEmail(hex); +} + +/** Removes the stored hashed email, for example on sign-out. */ +export function clearHashedEmail(): void { + writeCookie('', 0); +} diff --git a/crates/trusted-server-js/lib/src/core/index.ts b/crates/trusted-server-js/lib/src/core/index.ts index b2c4e41e15..a4910f8957 100644 --- a/crates/trusted-server-js/lib/src/core/index.ts +++ b/crates/trusted-server-js/lib/src/core/index.ts @@ -13,6 +13,7 @@ import { log } from './log'; import { setConfig, getConfig } from './config'; import { requestAds } from './request'; import { installQueue } from './queue'; +import { clearHashedEmail, setEmail, setHashedEmail } from './hashed_email'; const VERSION = '0.1.0'; @@ -35,6 +36,9 @@ api.setConfig = setConfig; api.getConfig = getConfig; // Provide core requestAds API api.requestAds = requestAds; +api.setEmail = setEmail; +api.setHashedEmail = setHashedEmail; +api.clearHashedEmail = clearHashedEmail; // Defensive defaults: the edge injects adSlots (head-open) and bids (before // ) only when server-side ad templates run for the request. When template // delivery is disabled or gated off (auction/consent, bots, prefetch), page code @@ -68,5 +72,8 @@ log.info('tsjs initialized', { 'addAdUnits', 'renderAdUnit', 'renderAllAdUnits', + 'setEmail', + 'setHashedEmail', + 'clearHashedEmail', ], }); diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 3407d487a0..e4aa3d9224 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -416,6 +416,16 @@ export interface TsjsApi { callback: () => void, opts?: { bidsBackHandler?: () => void; timeout?: number } ): void; + /** + * Normalizes and hashes the visitor's email in the browser and stores the + * SHA-256 in the first-party `ts-hem` cookie, the identity providers' + * hashed-email input. Resolves to whether a digest was stored. + */ + setEmail?(email: string): Promise; + /** Stores an already-computed SHA-256 (64 hex digits) in `ts-hem`. */ + setHashedEmail?(hex: string): boolean; + /** Removes `ts-hem`, for example on sign-out. */ + clearHashedEmail?(): void; log?: { setLevel(l: 'silent' | 'error' | 'warn' | 'info' | 'debug'): void; getLevel(): 'silent' | 'error' | 'warn' | 'info' | 'debug'; diff --git a/crates/trusted-server-js/lib/test/core/hashed_email.test.ts b/crates/trusted-server-js/lib/test/core/hashed_email.test.ts new file mode 100644 index 0000000000..359d1f8fa6 --- /dev/null +++ b/crates/trusted-server-js/lib/test/core/hashed_email.test.ts @@ -0,0 +1,143 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { + clearHashedEmail, + normalizeEmail, + setEmail, + setHashedEmail, +} from '../../src/core/hashed_email'; + +const EXAMPLE_HASH = 'ab'.repeat(32); + +describe('core/hashed_email', () => { + let writes: string[]; + + beforeEach(() => { + writes = []; + Object.defineProperty(document, 'cookie', { + configurable: true, + get: () => '', + set: (assignment: string) => { + writes.push(assignment); + }, + }); + }); + + afterEach(() => { + delete (document as unknown as { cookie?: string }).cookie; + }); + + it('normalizes case, whitespace and Gmail aliases', () => { + expect(normalizeEmail(' Person@Example.COM ')).toBe('person@example.com'); + expect(normalizeEmail('first.last+news@gmail.com')).toBe('firstlast@gmail.com'); + expect(normalizeEmail('first.last+news@example.com')).toBe('first.last+news@example.com'); + expect(normalizeEmail('not-an-address')).toBeUndefined(); + expect(normalizeEmail('@example.com')).toBeUndefined(); + }); + + it('stores a hex digest in ts-hem', () => { + expect(setHashedEmail(EXAMPLE_HASH.toUpperCase())).toBe(true); + expect(writes).toEqual([ + `ts-hem=${EXAMPLE_HASH}; Path=/; Secure; SameSite=Lax; Max-Age=2592000`, + ]); + }); + + it('refuses a value that is not a digest', () => { + expect(setHashedEmail('person@example.com')).toBe(false); + expect(writes).toEqual([]); + }); + + it('hashes the normalized email without storing the address', async () => { + const stored = await setEmail(' Person@Example.com '); + + const expected = Array.from( + new Uint8Array( + await crypto.subtle.digest('SHA-256', new TextEncoder().encode('person@example.com')) + ), + (byte) => byte.toString(16).padStart(2, '0') + ).join(''); + expect(stored).toBe(true); + expect(writes).toEqual([`ts-hem=${expected}; Path=/; Secure; SameSite=Lax; Max-Age=2592000`]); + expect(writes[0]).not.toContain('example.com'); + }); + + it('matches the published SHA-256 vector for a normalized address', async () => { + await setEmail('Test@Example.com'); + + expect(writes).toEqual([ + 'ts-hem=973dfe463ec85785f5f95af5ba3906eedb2d931c24e69824a89ea65dba4e813b; Path=/; Secure; SameSite=Lax; Max-Age=2592000', + ]); + }); + + it.each([ + ['googlemail aliases', 'J.Doe+promo@GoogleMail.com', 'jdoe@googlemail.com'], + ['a dotted Gmail local part', 'j.d.o.e@gmail.com', 'jdoe@gmail.com'], + ['several plus signs on Gmail', 'jdoe+a+b@gmail.com', 'jdoe@gmail.com'], + ['dots on another domain', 'j.doe@example.com', 'j.doe@example.com'], + ])('normalizes %s', (_label, input, expected) => { + expect(normalizeEmail(input)).toBe(expected); + }); + + it.each([ + ['an empty string', ''], + ['only whitespace', ' '], + ['a missing domain', 'person@'], + ['inner whitespace', 'per son@example.com'], + ['two at signs', 'a@b@example.com'], + ['a Gmail local part that is only a suffix', '+news@gmail.com'], + ['a Gmail local part that is only dots', '..@gmail.com'], + ])('refuses %s', (_label, input) => { + expect(normalizeEmail(input)).toBeUndefined(); + }); + + it('writes nothing for an address it refuses', async () => { + expect(await setEmail('not-an-address')).toBe(false); + expect(writes).toEqual([]); + }); + + it('writes nothing when Web Crypto is unavailable', async () => { + const original = Object.getOwnPropertyDescriptor(globalThis, 'crypto'); + Object.defineProperty(globalThis, 'crypto', { configurable: true, value: {} }); + try { + expect(await setEmail('person@example.com')).toBe(false); + } finally { + if (original) { + Object.defineProperty(globalThis, 'crypto', original); + } + } + expect(writes).toEqual([]); + }); + + it.each([ + ['63 digits', 'a'.repeat(63)], + ['65 digits', 'a'.repeat(65)], + ['a non-hex digit', 'g'.repeat(64)], + ['a cookie separator', `${'a'.repeat(63)};`], + ])('refuses a digest with %s', (_label, value) => { + expect(setHashedEmail(value)).toBe(false); + expect(writes).toEqual([]); + }); + + it('trims a digest before storing it', () => { + expect(setHashedEmail(` ${EXAMPLE_HASH}\n`)).toBe(true); + expect(writes[0]).toBe(`ts-hem=${EXAMPLE_HASH}; Path=/; Secure; SameSite=Lax; Max-Age=2592000`); + }); + + it('does not throw when the cookie cannot be written', () => { + Object.defineProperty(document, 'cookie', { + configurable: true, + get: () => '', + set: () => { + throw new Error('sandboxed'); + }, + }); + + expect(() => setHashedEmail(EXAMPLE_HASH)).not.toThrow(); + expect(() => clearHashedEmail()).not.toThrow(); + }); + + it('clears ts-hem', () => { + clearHashedEmail(); + expect(writes).toEqual(['ts-hem=; Path=/; Secure; SameSite=Lax; Max-Age=0']); + }); +}); diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index d4a41b2800..7bf9f4053a 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -169,6 +169,10 @@ export default withMermaid( text: 'GPT Runtime Diagnostics', link: '/guide/integrations/gpt-diagnostics', }, + { + text: 'Identity Vendors', + link: '/guide/integrations/identity-vendors', + }, { text: 'Lockr', link: '/guide/integrations/lockr' }, { text: 'Next.js', link: '/guide/integrations/nextjs' }, { text: 'Osano', link: '/guide/integrations/osano' }, diff --git a/docs/guide/identity-providers.md b/docs/guide/identity-providers.md index e93deefe8e..64b952efda 100644 --- a/docs/guide/identity-providers.md +++ b/docs/guide/identity-providers.md @@ -2,7 +2,7 @@ This guide is for engineers who add a vendor to Trusted Server's server-side identity framework. Operators configuring a provider that already exists want the [Configuration Reference](/guide/configuration#identity-providers-ec-identity) and the [Edge Cookies](/guide/edge-cookies) guide instead. -Every example uses a fictional provider, `exampleid`, that calls `api.exampleid.example.com`. No vendor ships with the framework. Vendors arrive in their own specs. +Every example uses a fictional provider, `exampleid`, that calls `api.exampleid.example.com`. The vendor providers that ship with Trusted Server, listed in [Identity Vendors](/guide/integrations/identity-vendors), are worked examples of the same API. For the protocol, the storage layout and the reasoning behind each rule, see the [design spec](https://github.com/IABTechLab/trusted-server/blob/main/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md). Section numbers below point into it. @@ -79,6 +79,10 @@ An accessor fails in one of two ways: - `UndeclaredInput`: you read something the descriptor never declared. This is a provider bug. Even if you hide the error with `.ok()`, the read is recorded, the coordinator logs it as `Config`, and the harness fails your test. - `InputUnavailable`: the input was declared but is missing this time. This is a normal skip, handled like returning `None` from `request`. +The hashed email (`Input::HashedEmail`) comes from the `ts-hem` cookie the page sets through the JS SDK, so it is present only for visitors who signed in. Treat it as wanted unless the vendor cannot work without it. + +A provider can also read a first-party cookie on the publisher's domain, such as the vendor's own ID cookie. Declare each name with `Resolves::with_cookie(name)` and read it with `input.cookie(name)?`, which returns `None` when the request lacks it. Trusted Server's own cookies and the consent cookies cannot be declared. Consent comes from `consent()`. + Always available: `consent()`, `publisher_domain()`, `now()`, `random_bytes()`, `stored(source)` and the declared `endpoint()`. When the sources in one request have different kinds, use `input.is_initial(&source)` or `input.is_refresh(&source)` rather than matching on the kind. ## Building a request diff --git a/docs/guide/integrations/identity-vendors.md b/docs/guide/integrations/identity-vendors.md new file mode 100644 index 0000000000..9ba42f7d2b --- /dev/null +++ b/docs/guide/integrations/identity-vendors.md @@ -0,0 +1,145 @@ +# Identity Vendors + +Trusted Server ships server-side [identity providers](/guide/identity-providers) for six vendors. Each one is an integration under `[integrations.]` that, once enabled, resolves or captures IDs at the edge and writes them to the Edge Cookie (EC) identity graph in KV. You don't need the vendor's browser module for this. + +| Integration | Vendor | Kind | Sources | Needs | +| ------------ | --------------- | ------- | --------------------------------------------------- | ------------------------------------- | +| `id5` | ID5 | Resolve | `id5-sync.com`, `true-link-id5-sync.com`, `euid.eu` | Nothing (hashed email improves match) | +| `liveintent` | LiveIntent | Resolve | One per configured attribute (see below) | Nothing (`_lc2_fpi` improves match) | +| `liveramp` | LiveRamp ATS | Resolve | `liveramp.com` | Hashed email for a first envelope | +| `uid2` | UID2 | Resolve | `uidapi.com` | Hashed email for a first token | +| `connectid` | Yahoo ConnectID | Resolve | `yahoo.com` | Hashed email | +| `lockr` | Lockr (capture) | Capture | `id5-sync.com`, `liveramp.com`, `uidapi.com` | Lockr's proxied API | + +Criteo and The Trade Desk are not included. Their IDs live in the vendor's own third-party cookie, which Trusted Server cannot read at the publisher's edge. Browser ingest still carries those IDs from `ts-eids` when they are registered partners. + +## Before you enable one + +Every source a provider writes needs an `[[ec.partners]]` entry, or `ts config validate` refuses the config. When a provider declares more sources than you want, narrow it: + +```toml +[[ec.partners]] +name = "ID5" +source_domain = "id5-sync.com" +openrtb_atype = 1 +bidstream_enabled = true + +[ec.identity.providers.id5] +sources = ["id5-sync.com"] +``` + +Provider timing, rate limits and ownership use the keys in the [Configuration Reference](/guide/configuration#identity-providers-ec-identity). + +## Hashed email + +LiveRamp, UID2 and ConnectID need a hashed email, and ID5 sends one when it is present. The page supplies it after the visitor signs in. The JS SDK normalizes and hashes the address in the browser and stores only the SHA-256 in the first-party `ts-hem` cookie: + +```js +window.tsjs.que.push(async () => { + await window.tsjs.setEmail('person@example.com') +}) + +// Or, when the page already has the digest: +window.tsjs.setHashedEmail('<64 hex digits>') + +// On sign-out: +window.tsjs.clearHashedEmail() +``` + +Normalization trims and lowercases the address. For Gmail it also drops dots and any `+suffix` from the local part. The cookie lasts 30 days. + +Trusted Server reads `ts-hem` on each request and passes it only to providers that declare the hashed-email input. It never stores the digest in KV, and it strips the cookie before forwarding a request to the origin or to Prebid Server. A hashed email is personal data: set it only under the legal basis your consent platform records. + +## ID5 + +```toml +[integrations.id5] +enabled = true +partner_id = 1234 +# endpoint = "https://id5-sync.com/g/v2" +``` + +ID5 needs vendor consent for GVL ID 131 where TCF applies, and consent for the purposes its GVL entry lists: 1, 3, 5 and 10. Trusted Server keeps the signature ID5 returns as private state and sends it on the next call. ID5's answer `0` (the visitor opted out at ID5) removes the stored ID. Confirm with ID5 the server-to-server field names for the client IP (`ip`) and user agent (`ua`) on your partner account. + +## LiveIntent + +```toml +[integrations.liveintent] +enabled = true +publisher_id = "12345" +# distributor_id = "did-0000" +resolve = ["nonId", "thetradedesk"] +``` + +`resolve` lists the attributes to ask for. Each attribute maps to one source: + +| Attribute | Source | +| -------------- | ------------------------------ | +| `nonId` | `liveintent.com` | +| `bidswitch` | `bidswitch.net` | +| `medianet` | `media.net` | +| `magnite` | `rubiconproject.com` | +| `index` | `liveintent.indexexchange.com` | +| `openx` | `openx.net` | +| `pubmatic` | `pubmatic.com` | +| `sovrn` | `liveintent.sovrn.com` | +| `sharethrough` | `sharethrough.com` | +| `sonobi` | `liveintent.sonobi.com` | +| `triplelift` | `liveintent.triplelift.com` | +| `zetassp` | `zeta-ssp.liveintent.com` | +| `vidazoo` | `liveintent.vidazoo.com` | +| `nexxen` | `liveintent.unrulymedia.com` | +| `uid2` | `uidapi.com` | +| `thetradedesk` | `adserver.org` | + +Only the configured attributes are declared, so only their sources need partner entries. LiveIntent needs vendor consent for GVL ID 148 and consent for Purposes 1, 2 and 7, as its GVL entry lists. When LiveIntent's first-party `_lc2_fpi` cookie is on the request, it is sent as `duid`. + +## LiveRamp ATS + +```toml +[integrations.liveramp] +enabled = true +placement_id = "placement-1" +approved_countries = ["US"] +# endpoint = "https://api.rlcdn.com/api/identity/v2/envelope" +``` + +A first envelope needs the hashed email. A refresh sends the stored envelope instead, so it works on later visits without one. Envelopes expire after 15 days in California and 30 days elsewhere. Calls run only in `approved_countries`, and need vendor consent for GVL ID 97 and consent for Purposes 1 to 10, as its GVL entry lists. The response shape and consent parameter names follow the design spec's assumptions; confirm them with LiveRamp. + +## UID2 + +```toml +[integrations.uid2] +enabled = true +# base_url = "https://prod.uidapi.com" +api_key = "uid2_api_key" +client_secret = "uid2_client_secret" +# tcf_vendor_id = 21 +``` + +`api_key` and `client_secret` are secret-store key names; `ts config push` resolves them. The client secret is the base64 key that encrypts token requests. A first token needs the hashed email. Trusted Server keeps the refresh token as private state and refreshes before the token expires. UID2 has no GVL ID: where TCF applies it needs Purposes 1 and 4, and vendor consent only when `tcf_vendor_id` names your UID2 operator's GVL ID. A UID2 `optout` opts the visitor out of UID2 until they consent again. UID2 is not meant for the EU (EUID covers it), so set `tcf_vendor_id` only when your UID2 agreement names one. + +## Yahoo ConnectID + +```toml +[integrations.connectid] +enabled = true +pixel_id = 12345 +# endpoint = "https://ups.analytics.yahoo.com/ups" +``` + +ConnectID needs the hashed email, vendor consent for GVL ID 25 and consent for Purposes 1 to 11, as Yahoo's GVL entry lists. Yahoo's `ttl`, in hours, sets when the ID is refreshed. + +## Lockr capture + +```toml +[integrations.lockr] +enabled = true +app_id = "app-1" +capture_identity = true + +[ec.identity.providers.lockr] +sources = ["id5-sync.com"] +``` + +With `capture_identity`, Lockr becomes a capture-only provider on the Lockr API calls Trusted Server already proxies. Capture is revoke-only: a `revoke-consent` call removes the IDs of the listed sources and blocks the browser's values for them until Lockr issues again. Lockr's IDs themselves reach KV through browser ingest. A capture-only provider must list its `sources`. diff --git a/docs/superpowers/plans/2026-10-28-identity-provider-connectid.md b/docs/superpowers/plans/2026-10-28-identity-provider-connectid.md new file mode 100644 index 0000000000..4f16f17777 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-identity-provider-connectid.md @@ -0,0 +1,538 @@ +# Yahoo ConnectID Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve Yahoo ConnectID at the edge with a server-side `IdentityProvider` under `[integrations.connectid]`, writing it to the `yahoo.com` source. + +**Architecture:** `integrations/connectid.rs` holds `ConnectIdConfig`, `ConnectIdProvider` and `register`. The provider sends one `GET` to Yahoo's user-profile sync endpoint (`//fed`) with the hashed email from `ts-hem`, the page URL and the consent strings. Yahoo returns the ConnectID and a lifetime in hours, which becomes the ID's refresh time. + +**Tech Stack:** Rust 1.95 (edition 2024), `trusted-server-core`, `serde`, `http`, `validator`, `error-stack` (all already dependencies). + +**Spec:** `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`, sections 4.2 to 4.6 (the provider API) and section 12 (vendor examples). Operator page: `docs/guide/integrations/identity-vendors.md`. + +**Prerequisite:** `docs/superpowers/plans/2026-10-28-identity-provider-inputs.md`. ConnectID needs `Input::HashedEmail` from the `ts-hem` cookie, so it is called only for visitors whose page set one. + +## Design notes + +- **The hashed email is needed** for every call (`with_needs([Input::HashedEmail])`); Yahoo has nothing to match on without it. It is sent as `he`, lowercase hex. +- **Two spellings of the ID.** Yahoo returns both `connectId` and `connectid` in one reply, often together. Two serde aliases on one field would fail such a reply as a duplicate field, so the reply type has two fields and the provider takes `connectId` first, then `connectid`. +- **`ttl` is in hours.** A positive `ttl` sets the refresh time; zero or absent leaves pacing to `refresh_after_sec`. +- A 204, an empty reply or an empty ID is no match. + +## Global Constraints + +- Follow `AGENTS.md`: `error-stack` with `derive_more::Display`, `expect("should ...")`, no local imports, comments above code, doc comment on every public item, Arrange-Act-Assert tests. +- Backtick `ConnectID` and other CamelCase names in doc comments (`clippy::doc_markdown`). +- Tests use fictional data: `ups.example.com` endpoint, `www.example.com` pages, fake IDs (`cid-example`). The public default endpoint host and the `yahoo.com` source may appear. +- Never log the hashed email, the ConnectID or consent strings. +- Yahoo's IAB GVL ID is **25**. UIDs use `atype` **3**. +- Quick test: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::connectid`. Gates: `cargo test-fastly`, `cargo clippy-fastly`. + +## File Structure + +| File | Change | Responsibility | +| ---------------------------------------------------------- | ------ | ------------------------------------------------------- | +| `crates/trusted-server-core/src/integrations/connectid.rs` | Create | Config, descriptor, provider, registration, tests | +| `crates/trusted-server-core/src/integrations/mod.rs` | Modify | `pub mod connectid;` and its `IntegrationBuilder` | +| `crates/trusted-server-core/src/config.rs` | Modify | Runtime and deploy validation, deploy-validated ID list | +| `docs/guide/integrations/identity-vendors.md` | Modify | Table row and `## Yahoo ConnectID` section | +| `CHANGELOG.md` | Modify | Vendor bullet | + +--- + +### Task 1: Config, descriptor and request + +**Files:** + +- Create: `crates/trusted-server-core/src/integrations/connectid.rs` +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (add `pub mod connectid;` in alphabetical order) + +**Interfaces:** + +- Consumes: `ResolveInput::{endpoint, consent, hashed_email, page_url}`, `HashedEmail::to_hex`, `ProviderRequest::{get, with_query}`. +- Produces: `pub(crate) const CONNECTID_INTEGRATION_ID: &str = "connectid"`, `pub struct ConnectIdConfig { enabled, pixel_id: u64, endpoint: String }`, `pub struct ConnectIdProvider`, `pub fn ConnectIdProvider::new(&ConnectIdConfig) -> Result>`. + +- [ ] **Step 1: Write the failing tests** + +Create `connectid.rs` with the tests module only: + +```rust +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::ec::identity::HashedEmail; + use crate::ec::identity::test_support::{ + ConformanceCases, HarnessSetup, ProviderHarness, check_descriptor_ignores_secrets, + }; + + fn config() -> ConnectIdConfig { + ConnectIdConfig { + enabled: true, + pixel_id: 58_776, + endpoint: "https://ups.example.com/ups".to_owned(), + } + } + + fn provider() -> ConnectIdProvider { + ConnectIdProvider::new(&config()).expect("should build the provider") + } + + #[test] + fn sends_the_hashed_email_to_the_pixel() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_hashed_email(HashedEmail::for_test([0xab; 32])) + .with_page_url("https://www.example.com/article"); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + assert_eq!( + request.url().path(), + "/ups/58776/fed", + "should address the pixel" + ); + let he = request + .url() + .query_pairs() + .find(|(key, _)| key == "he") + .map(|(_, value)| value.into_owned()); + assert_eq!(he, Some("ab".repeat(32)), "should send the HEM hex"); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets(&config(), &config()); + } +} +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::connectid` +Expected: FAIL to compile (`ConnectIdConfig` not found). + +- [ ] **Step 3: Implement the config and descriptor** + +Prepend to `connectid.rs`. The `use` block is the module's final one; items used only by Task 2 warn as unused until then. + +```rust +//! Yahoo `ConnectID` identity provider: resolves `ConnectID` server side. +//! +//! Calls Yahoo's user-profile sync endpoint with the hashed email from the +//! `ts-hem` cookie, the page and the consent strings. Yahoo returns the +//! `ConnectID` and a lifetime in hours. + +use std::time::Duration; + +use error_stack::Report; +use http::StatusCode; +use serde::Deserialize; +use validator::Validate; + +use crate::ec::identity::{ + ConsentPolicy, IdentityProvider, IdentityProviderConfig, IdentityProviderError, Input, + ProviderDescriptor, ProviderRequest, ProviderResponse, Resolution, ResolveInput, ResolvedEid, + ResolvedUid, Resolves, +}; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistration; +use crate::settings::{IntegrationConfig, Settings}; + +/// Integration ID of the `ConnectID` provider. +pub(crate) const CONNECTID_INTEGRATION_ID: &str = "connectid"; + +/// Yahoo's IAB Global Vendor List ID. +const YAHOO_GVL_ID: u16 = 25; + +const SECONDS_PER_HOUR: u64 = 3_600; + +/// Config of the `connectid` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct ConnectIdConfig { + /// Enables the provider. + #[serde(default)] + pub enabled: bool, + /// The pixel ID Yahoo assigned to the publisher. + pub pixel_id: u64, + /// The sync endpoint; the pixel ID and `fed` are appended to its path. + #[serde(default = "default_endpoint")] + #[validate(url)] + pub endpoint: String, +} + +fn default_endpoint() -> String { + "https://ups.analytics.yahoo.com/ups".to_owned() +} + +impl IntegrationConfig for ConnectIdConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl IdentityProviderConfig for ConnectIdConfig { + fn build_descriptor(&self) -> Result> { + ProviderDescriptor::builder(CONNECTID_INTEGRATION_ID.parse()?) + .with_source("yahoo.com".parse()?) + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_needs([Input::HashedEmail]) + .with_wants([Input::PageUrl]), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(YAHOO_GVL_ID)) + .build() + } +} +``` + +- [ ] **Step 4: Implement the provider and its request** + +Append, before the tests module: + +```rust +/// The Yahoo `ConnectID` provider. +pub struct ConnectIdProvider { + descriptor: ProviderDescriptor, + pixel_id: String, +} + +impl ConnectIdProvider { + /// Builds the provider from its validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config cannot + /// describe a valid provider. + pub fn new(config: &ConnectIdConfig) -> Result> { + Ok(Self { + descriptor: config.build_descriptor()?, + pixel_id: config.pixel_id.to_string(), + }) + } +} + +impl IdentityProvider for ConnectIdProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let mut url = input.endpoint()?.clone(); + url.path_segments_mut() + .map_err(|_| IdentityProviderError::config("endpoint cannot hold a path"))? + .pop_if_empty() + .push(&self.pixel_id) + .push("fed"); + let consent = input.consent(); + let mut request = ProviderRequest::get(&url) + .with_query("v", "1") + .with_query("1p", "0") + .with_query("pixelId", &self.pixel_id) + .with_query("he", &input.hashed_email()?.to_hex()) + .with_query("gdpr", if consent.gdpr_applies() { "1" } else { "0" }); + if let Some(tc_string) = consent.raw_tc_string() { + request = request.with_query("gdpr_consent", tc_string); + } + if let Some(us_privacy) = consent.raw_us_privacy() { + request = request.with_query("us_privacy", us_privacy); + } + if let Some(gpp) = consent.raw_gpp_string() { + request = request.with_query("gpp", gpp); + } + if let Some(ids) = consent.gpp_section_ids() { + let ids: Vec = ids.iter().map(u16::to_string).collect(); + request = request.with_query("gpp_sid", &ids.join(",")); + } + if let Ok(page) = input.page_url() { + request = request.with_query("url", page.as_str()); + } + Ok(Some(request)) + } +} +``` + +The page URL the framework hands over has no query or fragment, so `url` never carries page parameters. + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::connectid` +Expected: PASS (2 tests). + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/connectid.rs crates/trusted-server-core/src/integrations/mod.rs +git commit -S --signoff -m "Build Yahoo ConnectID sync requests from the hashed email" +``` + +--- + +### Task 2: Parse the ConnectID and its lifetime + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/connectid.rs` + +**Interfaces:** + +- Consumes: `ProviderResponse::{status, success_json}`, `ResolvedEid::with_refresh_in`. +- Produces: `ConnectIdProvider::response`. + +- [ ] **Step 1: Write the failing tests** + +Add to the tests module: + +```rust + #[test] + fn resolves_the_connect_id_with_its_ttl() { + let provider = provider(); + let mut harness = + ProviderHarness::new(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "connectId": "cid-example", "connectid": "cid-example", "ttl": 24 }), + ) + .expect("should parse"); + + let eid = &resolution.eids()[0]; + assert_eq!(eid.source().as_str(), "yahoo.com", "should use yahoo.com"); + assert_eq!(eid.uids()[0].id(), "cid-example", "should keep the ID"); + assert!(eid.refresh().is_some(), "should set a refresh from the ttl"); + } + + #[test] + fn an_empty_reply_is_no_match() { + let provider = provider(); + let mut harness = + ProviderHarness::new(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let resolution = harness + .respond(StatusCode::OK, &json!({})) + .expect("should parse"); + + assert!(resolution.is_no_match(), "should be no match"); + } + + #[test] + fn conforms() { + ProviderHarness::new(&provider()).check_conformance( + &ConformanceCases::default().with_resolve_case( + HarnessSetup::default().with_hashed_email(HashedEmail::for_test([0xab; 32])), + StatusCode::OK, + &json!({ "connectId": "cid-example" }), + ), + ); + } +``` + +The first test sends both spellings in one reply, as Yahoo does, which is the case a two-alias field would fail. + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::connectid` +Expected: the three new tests FAIL with `NotImplemented("response")`. + +- [ ] **Step 3: Add the reply type** + +Append after `impl ConnectIdProvider`: + +```rust +#[derive(Deserialize)] +struct SyncReply { + #[serde(rename = "connectId")] + connect_id: Option, + /// The lowercase spelling Yahoo also sends, often next to `connectId`. + connectid: Option, + /// Hours. + ttl: Option, +} +``` + +- [ ] **Step 4: Implement `response`** + +Add to `impl IdentityProvider for ConnectIdProvider`, after `request`: + +```rust + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + if response.status() == StatusCode::NO_CONTENT { + return Ok(Resolution::no_match()); + } + let reply: SyncReply = response.success_json()?; + // An empty `connectId` must not hide a usable `connectid`. + let Some(connect_id) = [reply.connect_id, reply.connectid] + .into_iter() + .flatten() + .find(|id| !id.is_empty()) + else { + return Ok(Resolution::no_match()); + }; + let mut eid = ResolvedEid::new( + input.primary_source(), + ResolvedUid::new(connect_id).with_atype(3), + ); + if let Some(hours) = reply.ttl.filter(|hours| *hours > 0) { + eid = eid.with_refresh_in(Duration::from_secs(hours.saturating_mul(SECONDS_PER_HOUR))); + } + Ok(Resolution::found([eid])) + } +``` + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::connectid` +Expected: PASS (5 tests). + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/connectid.rs +git commit -S --signoff -m "Resolve the Yahoo ConnectID with its hour-based lifetime" +``` + +--- + +### Task 3: Register, validate and document + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/connectid.rs` (`register`) +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (`builders()`) +- Modify: `crates/trusted-server-core/src/config.rs` +- Modify: `docs/guide/integrations/identity-vendors.md`, `CHANGELOG.md` + +**Interfaces:** + +- Produces: `pub fn register(settings: &Settings) -> Result, Report>`. + +- [ ] **Step 1: Write `register` and add the builder** + +Append to `connectid.rs`, before the tests module: + +```rust +/// Registers the `ConnectID` provider when `[integrations.connectid]` is +/// enabled. +/// +/// # Errors +/// +/// Returns an error when the config does not parse or describes an invalid +/// provider. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.integration_config::(CONNECTID_INTEGRATION_ID)? + else { + return Ok(None); + }; + log::info!( + "Registering Yahoo ConnectID identity provider (pixel={})", + config.pixel_id + ); + Ok(Some( + IntegrationRegistration::builder(CONNECTID_INTEGRATION_ID) + .try_with_identity_provider(ConnectIdProvider::new(&config))? + .without_js() + .build(), + )) +} +``` + +In `integrations/mod.rs` `builders()`, add: + +```rust + IntegrationBuilder { + id: connectid::CONNECTID_INTEGRATION_ID, + build: connectid::register, + }, +``` + +- [ ] **Step 2: Run the deploy-coverage test and see it fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config::tests::deploy_validation_covers_registered_integration_builders` +Expected: FAIL with `missing_ids` containing `"connectid"`. + +- [ ] **Step 3: Validate at runtime and at deploy** + +In `config.rs`, import `connectid::{CONNECTID_INTEGRATION_ID, ConnectIdConfig}`. Add `CONNECTID_INTEGRATION_ID,` to `DEPLOY_VALIDATED_INTEGRATION_IDS`. In `identity_descriptors_for_deploy`: + +```rust + descriptors.extend(validate_identity_integration::( + settings, + CONNECTID_INTEGRATION_ID, + )?); +``` + +If `validate_identity_integration` still carries the framework's temporary `#[allow(dead_code)]`, remove it and its comment. In `validate_enabled_integrations`: + +```rust + validate_integration::(settings, CONNECTID_INTEGRATION_ID)?; +``` + +- [ ] **Step 4: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config` +Expected: PASS. + +- [ ] **Step 5: Document** + +In `docs/guide/integrations/identity-vendors.md`, add this table row: + +```markdown +| `connectid` | Yahoo ConnectID | Resolve | `yahoo.com` | Hashed email | +``` + +and this section: + +````markdown +## Yahoo ConnectID + +```toml +[integrations.connectid] +enabled = true +pixel_id = 12345 +# endpoint = "https://ups.analytics.yahoo.com/ups" +``` + +ConnectID needs the hashed email and vendor consent for GVL ID 25. Yahoo's `ttl`, in hours, sets when the ID is refreshed. +```` + +In `CHANGELOG.md` under `### Added`, insert "Yahoo ConnectID (`[integrations.connectid]`)" into the vendor bullet's list of providers, or, if there is no vendor bullet yet, add: + +```markdown +- Server-side identity provider for Yahoo ConnectID (`[integrations.connectid]`). Each source it writes needs an `[[ec.partners]]` entry. See [Identity Vendors](docs/guide/integrations/identity-vendors.md). +``` + +- [ ] **Step 6: Run the gates** + +Run: `cargo fmt --all -- --check && cargo test-fastly && cargo clippy-fastly` +Run: `cd docs && npm run format` +Run: `docs/node_modules/.bin/prettier --config docs/.prettierrc --check CHANGELOG.md` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/ docs/guide/integrations/identity-vendors.md CHANGELOG.md +git commit -S --signoff -m "Register the Yahoo ConnectID identity provider and validate its config" +``` + +## Self-Review Notes + +- Hashed email needed, sent as hex `he`: Task 1. +- `connectId` and `connectid` as two fields, `ttl` in hours, 204 and empty no match: Task 2. diff --git a/docs/superpowers/plans/2026-10-28-identity-provider-id5.md b/docs/superpowers/plans/2026-10-28-identity-provider-id5.md new file mode 100644 index 0000000000..039757b342 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-identity-provider-id5.md @@ -0,0 +1,734 @@ +# ID5 Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve the ID5 ID at the edge with a server-side `IdentityProvider` under `[integrations.id5]`, storing the complete EIDs ID5 returns for `id5-sync.com`, `true-link-id5-sync.com` and `euid.eu`. + +**Architecture:** `integrations/id5.rs` holds `Id5Config` (an `IdentityProviderConfig`), `Id5Provider` (an `IdentityProvider`) and `register`. The provider is pure: `request` builds one JSON `POST` to ID5's fetch endpoint from the declared inputs, and `response` turns ID5's reply into a `Resolution`. ID5's `signature` is kept as private provider state and sent back on the next call. The coordinator from the framework handles consent, eligibility, timing, backoff and the KV write. + +**Tech Stack:** Rust 1.95 (edition 2024), `trusted-server-core`, `serde`/`serde_json`, `base64`, `url`, `validator`, `error-stack` (all already dependencies). + +**Spec:** `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`, section 12.4 (ID5) and sections 4.2 to 4.6 (the provider API). Operator page: `docs/guide/integrations/identity-vendors.md`. Author guide: `docs/guide/identity-providers.md`. + +**Prerequisite:** `docs/superpowers/plans/2026-10-28-identity-provider-inputs.md`. This plan uses `Input::HashedEmail` from the `ts-hem` cookie, `VendorEid` (`integrations/vendor_eid.rs`) and `is_routable_ip` (`integrations/vendor_request.rs`), and adds its section to the operator page that plan creates. + +## Why the request looks the way it does + +These were checked against ID5's live endpoints: + +- **Fetch URL `https://id5-sync.com/g/v2/.json`.** ID5's config endpoint returns this URL for a partner. The newer `gm/v3` endpoint expects a multiplexed body with a `requests` array and answers 400 to a flat body, so the provider uses `g/v2` with a flat JSON body. The configured `endpoint` is a base; `.json` is appended as a path segment. +- **`Origin` header.** ID5 checks the request origin, so the provider sends `Origin: https://`. +- **Response.** ID5 returns complete EIDs under `ids..eid`, a `signature`, and `cache_control.max_age_sec`. Older responses carry only `universal_uid` (with `ext.linkType`), so that is a fallback for the primary source. A `universal_uid` of `0` means the visitor opted out at ID5, so the stored ID is invalidated. +- **Hashed email.** ID5 takes it as partner data: `pd` is base64 of `1=`. +- **Client IP and user agent** are sent as `ip` and `ua`; the IP only when it is routable (see `is_routable_ip`). The operator doc asks operators to confirm those field names with ID5 for each partner account. + +## Global Constraints + +- Follow `AGENTS.md`: `error-stack` with `derive_more::Display`, `expect("should ...")`, no local imports, comments above code, doc comment on every public item, Arrange-Act-Assert tests. +- Backtick `OpenRTB` and CamelCase vendor names in doc comments (`clippy::doc_markdown`). +- Tests use fictional data: `www.example.com` pages, documentation IPs (`192.0.2.0/24`), fake IDs such as `ID5*example`. The vendor's public hostname `id5-sync.com` and its source domains may appear. +- Never log UIDs, the signature, the hashed email or consent strings. +- ID5's IAB GVL ID is **131**. Sources, in order: `id5-sync.com` (primary), `true-link-id5-sync.com`, `euid.eu`. +- Quick test: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::id5`. Gates: `cargo test-fastly`, `cargo clippy-fastly`. + +## File Structure + +| File | Change | Responsibility | +| ---------------------------------------------------- | ------ | ------------------------------------------------------- | +| `crates/trusted-server-core/src/integrations/id5.rs` | Create | Config, descriptor, provider, registration, tests | +| `crates/trusted-server-core/src/integrations/mod.rs` | Modify | `pub mod id5;` and its `IntegrationBuilder` | +| `crates/trusted-server-core/src/config.rs` | Modify | Runtime and deploy validation, deploy-validated ID list | +| `docs/guide/integrations/identity-vendors.md` | Modify | Table row and `## ID5` section | +| `CHANGELOG.md` | Modify | Vendor bullet | + +--- + +### Task 1: Config, descriptor and request + +**Files:** + +- Create: `crates/trusted-server-core/src/integrations/id5.rs` +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (add `pub mod id5;` in alphabetical order) + +**Interfaces:** + +- Consumes: `is_routable_ip(IpAddr) -> bool`, `ResolveInput::{consent, page_url, hashed_email, user_agent, client_ip, state, endpoint, publisher_domain}`, `ProviderRequest::post_json`, `try_with_header`. +- Produces: `pub(crate) const ID5_INTEGRATION_ID: &str = "id5"`, `pub struct Id5Config { enabled, partner_id: u32, endpoint: String }`, `pub struct Id5Provider`, `pub fn Id5Provider::new(&Id5Config) -> Result>`, `pub(crate) struct Id5State { signature: String }`. + +- [ ] **Step 1: Write the failing tests** + +Create `id5.rs` with the tests module only: + +```rust +#[cfg(test)] +mod tests { + use http::StatusCode; + + use super::*; + use crate::ec::identity::HashedEmail; + use crate::ec::identity::test_support::{ + ConformanceCases, HarnessSetup, ProviderHarness, TestConsent, + check_descriptor_ignores_secrets, + }; + + fn config() -> Id5Config { + Id5Config { + enabled: true, + partner_id: 1234, + endpoint: default_endpoint(), + } + } + + fn provider() -> Id5Provider { + Id5Provider::new(&config()).expect("should build the provider") + } + + #[test] + fn sends_consent_page_and_hashed_email() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_user_agent("ExampleBrowser/1.0") + .with_page_url("https://www.example.com/article") + .with_hashed_email(HashedEmail::for_test([0xab; 32])) + .with_consent(TestConsent::default().with_us_privacy("1YNN")) + .with_state(&Id5State { + signature: "prior-signature".to_owned(), + }); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + assert_eq!( + request.url().as_str(), + "https://id5-sync.com/g/v2/1234.json", + "should call the partner's fetch URL" + ); + let body: serde_json::Value = harness.request_json(); + + assert_eq!(body["partner"], 1234, "should send the partner"); + assert_eq!(body["gdpr"], 0, "should send that GDPR does not apply"); + assert_eq!(body["us_privacy"], "1YNN", "should send US privacy"); + assert_eq!( + body["s"], "prior-signature", + "should send the stored signature" + ); + assert_eq!( + body["u"], "https://www.example.com/article", + "should send the page" + ); + assert_eq!(body["ip"], "192.0.2.10", "should send the client IP"); + assert_eq!( + body["pd"], + STANDARD.encode(format!("1={}", "ab".repeat(32))), + "should send the hashed email as partner data" + ); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets(&config(), &config()); + } +} +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::id5` +Expected: FAIL to compile (`Id5Config`, `Id5Provider`, `Id5State` not found). + +- [ ] **Step 3: Write the config and descriptor** + +Prepend to `id5.rs`. The `use` block is the module's final one; items used only by Task 2 (`BTreeMap`, `Duration`, `json!`, `ResolvedEid`, `ResolvedUid`, `ProviderResponse`, `Resolution`, `VendorEid`) warn as unused until then. + +```rust +//! ID5 identity provider: resolves the ID5 ID server side. +//! +//! Calls ID5's fetch endpoint with the request's consent, page and network +//! signals, and the hashed email when the JS SDK set one. ID5 returns either +//! complete EIDs under `ids` or the legacy `universal_uid`. The signature ID5 +//! returns is kept as private state and sent on the next call. +//! +//! The request field names for the client IP and user agent follow ID5's +//! server-to-server integration and should be confirmed with ID5 for each +//! partner account. + +use std::collections::BTreeMap; +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use error_stack::Report; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use url::Url; +use validator::Validate; + +use crate::ec::identity::{ + ConsentPolicy, IdentityProvider, IdentityProviderConfig, IdentityProviderError, Input, + ProviderDescriptor, ProviderRequest, ProviderResponse, Resolution, ResolveInput, ResolvedEid, + ResolvedUid, Resolves, +}; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistration; +use crate::integrations::vendor_eid::VendorEid; +use crate::integrations::vendor_request::is_routable_ip; +use crate::settings::{IntegrationConfig, Settings}; + +/// Integration ID of the ID5 provider. +pub(crate) const ID5_INTEGRATION_ID: &str = "id5"; + +/// ID5's IAB Global Vendor List ID. +const ID5_GVL_ID: u16 = 131; + +/// The sources ID5 can return. Operators narrow them with +/// `[ec.identity.providers.id5].sources`. +const ID5_SOURCES: &[&str] = &["id5-sync.com", "true-link-id5-sync.com", "euid.eu"]; + +/// The universal UID ID5 returns for a visitor who opted out at ID5. +const ID5_OPTED_OUT_UID: &str = "0"; + +/// Config of the `id5` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct Id5Config { + /// Enables the provider. + #[serde(default)] + pub enabled: bool, + /// The partner number ID5 assigned to the publisher. + pub partner_id: u32, + /// The fetch endpoint; `.json` is appended to its path. + #[serde(default = "default_endpoint")] + #[validate(url)] + pub endpoint: String, +} + +fn default_endpoint() -> String { + "https://id5-sync.com/g/v2".to_owned() +} + +impl IntegrationConfig for Id5Config { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl IdentityProviderConfig for Id5Config { + fn build_descriptor(&self) -> Result> { + let mut builder = ProviderDescriptor::builder(ID5_INTEGRATION_ID.parse()?); + for source in ID5_SOURCES { + builder = builder.with_source(source.parse()?); + } + builder + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_wants([ + Input::ClientIp, + Input::UserAgent, + Input::PageUrl, + Input::HashedEmail, + ]) + .with_private_state(), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(ID5_GVL_ID)) + .build() + } +} +``` + +Every input is _wanted_, not _needed_: ID5 matches on whatever it gets, so a missing input never stops the call. + +- [ ] **Step 4: Write the provider and its request** + +Append (before the tests module): + +```rust +/// The ID5 provider. +pub struct Id5Provider { + descriptor: ProviderDescriptor, + partner_id: u32, +} + +impl Id5Provider { + /// Builds the provider from its validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config cannot + /// describe a valid provider. + pub fn new(config: &Id5Config) -> Result> { + Ok(Self { + descriptor: config.build_descriptor()?, + partner_id: config.partner_id, + }) + } +} + +/// The signature ID5 returns, sent back on the next call. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct Id5State { + signature: String, +} + +#[derive(Serialize)] +struct FetchBody<'a> { + partner: u32, + v: &'static str, + o: &'static str, + #[serde(rename = "nbPage")] + nb_page: u32, + gdpr: u8, + #[serde(skip_serializing_if = "Option::is_none")] + gdpr_consent: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + us_privacy: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + gpp_string: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + gpp_sid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + s: Option, + #[serde(skip_serializing_if = "Option::is_none")] + u: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + tml: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pd: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ua: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + ip: Option, +} + +impl IdentityProvider for Id5Provider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let consent = input.consent(); + let page = input.page_url().ok().map(Url::as_str); + let body = FetchBody { + partner: self.partner_id, + v: env!("CARGO_PKG_VERSION"), + o: "trusted-server", + nb_page: 1, + gdpr: u8::from(consent.gdpr_applies()), + gdpr_consent: consent.raw_tc_string(), + us_privacy: consent.raw_us_privacy(), + gpp_string: consent.raw_gpp_string(), + gpp_sid: consent.gpp_section_ids().map(join_ids), + s: input.state::()?.map(|state| state.signature), + u: page, + tml: page, + pd: input + .hashed_email() + .ok() + .map(|email| STANDARD.encode(format!("1={}", email.to_hex()))), + ua: input.user_agent().ok(), + ip: input + .client_ip() + .ok() + .filter(|ip| is_routable_ip(*ip)) + .map(|ip| ip.to_string()), + }; + let mut url = input.endpoint()?.clone(); + url.path_segments_mut() + .map_err(|_| IdentityProviderError::config("endpoint cannot hold a path"))? + .pop_if_empty() + .push(&format!("{}.json", self.partner_id)); + ProviderRequest::post_json(&url, &body)? + .try_with_header("origin", &format!("https://{}", input.publisher_domain())) + .map(Some) + } +} + +fn join_ids(ids: &[u16]) -> String { + ids.iter().map(u16::to_string).collect::>().join(",") +} +``` + +`input.hashed_email().ok()` and the other `.ok()` calls are safe for wanted inputs: the accessor returns `InputUnavailable` when the input is absent this time, which is a normal skip of that field. `pop_if_empty()` keeps a configured endpoint with a trailing slash from producing `//1234.json`. + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::id5` +Expected: PASS (2 tests). + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/id5.rs crates/trusted-server-core/src/integrations/mod.rs +git commit -S --signoff -m "Build ID5 fetch requests from consent, page and hashed email" +``` + +--- + +### Task 2: Parse ID5's response + +ID5's reply can hold complete EIDs for several sources. Each `ids..eid` whose `source` names a declared source becomes one `ResolvedEid`, with every UID and EID-level field kept (`VendorEid::to_resolved`); others are ignored. When no EID came back for the primary source, the legacy `universal_uid` fills it in, with `atype` 1 and the reply's `ext.linkType`. `0` there means an ID5 opt-out and invalidates the primary source. `cache_control.max_age_sec` sets each EID's refresh time. A declared source that has a stored value but was not returned this time is invalidated with `also_invalidate`, so a stale TrueLink or EUID value does not outlive ID5's answer. The new `signature` replaces the stored state. + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/id5.rs` + +**Interfaces:** + +- Consumes: `VendorEid::to_resolved(&SourceDomain) -> Option`, `ProviderDescriptor::source_named`, `ResolveInput::{primary_source, sources, stored}`, `Resolution::{found, no_match, invalidate, also_invalidate, replacing_state}`. +- Produces: `Id5Provider::response`. + +- [ ] **Step 1: Write the failing tests** + +Add to the tests module: + +```rust + #[test] + fn resolves_complete_eids_and_keeps_the_signature() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ + "signature": "next-signature", + "ids": { + "id5": { "eid": { + "source": "id5-sync.com", + "uids": [{ "id": "ID5*example", "atype": 1, "ext": { "linkType": 2 } }] + }}, + "trueLinkId": { "eid": { + "source": "true-link-id5-sync.com", + "uids": [{ "id": "true-link-example", "atype": 1 }] + }}, + "other": { "eid": { + "source": "other.example.com", + "uids": [{ "id": "ignored" }] + }} + }, + "cache_control": { "max_age_sec": 7200 } + }), + ) + .expect("should parse"); + + let sources: Vec<&str> = resolution + .eids() + .iter() + .map(|eid| eid.source().as_str()) + .collect(); + assert_eq!( + sources, + ["id5-sync.com", "true-link-id5-sync.com"], + "should keep only declared sources" + ); + let state: Id5State = harness.state_of(&resolution).expect("should replace state"); + assert_eq!( + state.signature, "next-signature", + "should keep the signature" + ); + } + + #[test] + fn falls_back_to_the_universal_uid() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "universal_uid": "ID5*legacy", "ext": { "linkType": 1 } }), + ) + .expect("should parse"); + + let eid = &resolution.eids()[0]; + assert_eq!( + eid.source().as_str(), + "id5-sync.com", + "should use the primary source" + ); + assert_eq!(eid.uids()[0].id(), "ID5*legacy", "should keep the UID"); + assert_eq!( + eid.uids()[0].ext(), + Some(&json!({ "linkType": 1 })), + "should keep the link type" + ); + } + + #[test] + fn an_opted_out_visitor_invalidates_the_id() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider); + harness.request().expect("should build a request"); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "universal_uid": "0" })) + .expect("should parse"); + + assert_eq!( + resolution.invalidated()[0].as_str(), + "id5-sync.com", + "should invalidate the ID5 ID" + ); + } + + #[test] + fn conforms() { + ProviderHarness::new(&provider()).check_conformance_with_state::( + &ConformanceCases::default().with_resolve_case( + HarnessSetup::default().with_client_ip("192.0.2.10"), + StatusCode::OK, + &json!({ "universal_uid": "ID5*example", "signature": "s" }), + ), + ); + } +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::id5` +Expected: the four new tests FAIL with `NotImplemented("response")` from the trait's default `response`. + +- [ ] **Step 3: Add the reply types** + +Append after `FetchBody`: + +```rust +#[derive(Deserialize)] +struct FetchReply { + universal_uid: Option, + signature: Option, + #[serde(default)] + ext: Option, + #[serde(default)] + ids: BTreeMap, + cache_control: Option, +} + +#[derive(Deserialize)] +struct Id5Id { + eid: Option, +} + +#[derive(Deserialize)] +struct CacheControl { + max_age_sec: Option, +} +``` + +None of them denies unknown fields: ID5 adds fields over time and a new one must not fail the call. + +- [ ] **Step 4: Implement `response`** + +Add to `impl IdentityProvider for Id5Provider`, after `request`: + +```rust + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let reply: FetchReply = response.success_json()?; + let primary = input.primary_source(); + + let mut eids: Vec = reply + .ids + .values() + .filter_map(|id| { + let eid = id.eid.as_ref()?; + let source = self.descriptor.source_named(&eid.source)?; + eid.to_resolved(source) + }) + .collect(); + if !eids.iter().any(|eid| eid.source() == primary) { + match reply.universal_uid.as_deref() { + Some(ID5_OPTED_OUT_UID) => return Ok(Resolution::invalidate(primary)), + Some(uid) if !uid.is_empty() => { + let mut resolved = ResolvedUid::new(uid).with_atype(1); + if let Some(link_type) = reply.ext.as_ref().and_then(|ext| ext.get("linkType")) + { + resolved = resolved.with_ext(json!({ "linkType": link_type })); + } + eids.push(ResolvedEid::new(primary, resolved)); + } + _ => {} + } + } + if let Some(max_age) = reply.cache_control.and_then(|cache| cache.max_age_sec) { + eids = eids + .into_iter() + .map(|eid| eid.with_refresh_in(Duration::from_secs(max_age))) + .collect(); + } + + let mut resolution = if eids.is_empty() { + Resolution::no_match() + } else { + let returned: Vec<_> = eids.iter().map(|eid| eid.source().clone()).collect(); + let mut found = Resolution::found(eids); + for (source, _) in input.sources() { + if !returned.contains(source) && input.stored(source).is_some() { + found = found.also_invalidate(source); + } + } + found + }; + if let Some(signature) = reply.signature { + resolution = resolution.replacing_state(&Id5State { signature }); + } + Ok(resolution) + } +``` + +`success_json` maps a non-2xx status to `Rejected` and an unparseable body to `Malformed`, so the coordinator applies the framework's backoff for those. + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::id5` +Expected: PASS (6 tests). `check_conformance_with_state` exercises declared-input discipline, state round-trips, oversized and malformed bodies, and error statuses. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/id5.rs +git commit -S --signoff -m "Resolve complete ID5 EIDs and keep the ID5 signature" +``` + +--- + +### Task 3: Register, validate and document + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/id5.rs` (`register`) +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (`builders()`) +- Modify: `crates/trusted-server-core/src/config.rs` (imports, `DEPLOY_VALIDATED_INTEGRATION_IDS`, `identity_descriptors_for_deploy`, `validate_enabled_integrations`) +- Modify: `docs/guide/integrations/identity-vendors.md`, `CHANGELOG.md` + +**Interfaces:** + +- Consumes: `IntegrationRegistration::builder(..).try_with_identity_provider(..).without_js()`, `validate_identity_integration::(settings, id)`, `validate_integration::(settings, id)`. +- Produces: `pub fn register(settings: &Settings) -> Result, Report>`. + +- [ ] **Step 1: Write `register` and add the builder** + +Append to `id5.rs`, before the tests module: + +```rust +/// Registers the ID5 provider when `[integrations.id5]` is enabled. +/// +/// # Errors +/// +/// Returns an error when the config does not parse or describes an invalid +/// provider. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.integration_config::(ID5_INTEGRATION_ID)? else { + return Ok(None); + }; + log::info!( + "Registering ID5 identity provider (partner={})", + config.partner_id + ); + Ok(Some( + IntegrationRegistration::builder(ID5_INTEGRATION_ID) + .try_with_identity_provider(Id5Provider::new(&config))? + .without_js() + .build(), + )) +} +``` + +`without_js()` because the provider ships no browser module. In `integrations/mod.rs` `builders()`, add: + +```rust + IntegrationBuilder { + id: id5::ID5_INTEGRATION_ID, + build: id5::register, + }, +``` + +- [ ] **Step 2: Run the deploy-coverage test and see it fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config::tests::deploy_validation_covers_registered_integration_builders` +Expected: FAIL with `missing_ids` containing `"id5"`. This test exists so no registered integration escapes `ts config validate`. + +- [ ] **Step 3: Validate at runtime and at deploy** + +In `config.rs`, import `id5::{ID5_INTEGRATION_ID, Id5Config}` in the `crate::integrations` use list. Add `ID5_INTEGRATION_ID,` to `DEPLOY_VALIDATED_INTEGRATION_IDS`. In `identity_descriptors_for_deploy`, after the generic pull descriptors: + +```rust + descriptors.extend(validate_identity_integration::( + settings, + ID5_INTEGRATION_ID, + )?); +``` + +Deploy validation then cross-checks the ID5 descriptor's sources against `[[ec.partners]]` and `[ec.identity.providers.id5]`, as for every provider. If `validate_identity_integration` still carries `#[allow(dead_code)]` from the framework ("the first production identity integration adds the first caller"), remove that attribute and its comment now. In `validate_enabled_integrations`, add: + +```rust + validate_integration::(settings, ID5_INTEGRATION_ID)?; +``` + +- [ ] **Step 4: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config` +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::id5` +Expected: PASS. + +- [ ] **Step 5: Document** + +In `docs/guide/integrations/identity-vendors.md`, add this row to the vendor table: + +```markdown +| `id5` | ID5 | Resolve | `id5-sync.com`, `true-link-id5-sync.com`, `euid.eu` | Nothing (hashed email improves match) | +``` + +and this section after "Hashed email" (keep vendor sections in table order): + +````markdown +## ID5 + +```toml +[integrations.id5] +enabled = true +partner_id = 1234 +# endpoint = "https://id5-sync.com/g/v2" +``` + +ID5 needs vendor consent for GVL ID 131 where TCF applies. Trusted Server keeps the signature ID5 returns as private state and sends it on the next call. ID5's answer `0` (the visitor opted out at ID5) removes the stored ID. Confirm with ID5 the server-to-server field names for the client IP (`ip`) and user agent (`ua`) on your partner account. +```` + +In `CHANGELOG.md` under `### Added`, add ID5 to the vendor bullet. If no vendor bullet exists yet, add: + +```markdown +- Server-side identity provider for ID5 (`[integrations.id5]`). Each source it writes needs an `[[ec.partners]]` entry. See [Identity Vendors](docs/guide/integrations/identity-vendors.md). +``` + +When other vendor plans have already added the bullet, insert "ID5 (`[integrations.id5]`)" into its list of providers instead. + +- [ ] **Step 6: Run the gates** + +Run: `cargo fmt --all -- --check && cargo test-fastly && cargo clippy-fastly` +Run: `cd docs && npm run format` +Run: `docs/node_modules/.bin/prettier --config docs/.prettierrc --check CHANGELOG.md` +Expected: PASS, no unused-import warnings left. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/ docs/guide/integrations/identity-vendors.md CHANGELOG.md +git commit -S --signoff -m "Register the ID5 identity provider and validate its config" +``` + +## Self-Review Notes + +- Fetch URL, flat body and `Origin`: Task 1, with the `gm/v3` reason above. +- Complete EIDs, legacy fallback, `0` opt-out, `max_age_sec` refresh, `also_invalidate`, signature state: Task 2. +- Registration, runtime and deploy validation, operator doc: Task 3. diff --git a/docs/superpowers/plans/2026-10-28-identity-provider-inputs.md b/docs/superpowers/plans/2026-10-28-identity-provider-inputs.md new file mode 100644 index 0000000000..0cb9ce6eee --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-identity-provider-inputs.md @@ -0,0 +1,1384 @@ +# Identity Provider Inputs Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give identity providers the two request inputs the vendor providers need: a hashed email (HEM) that the page supplies through the first-party `ts-hem` cookie, and a declared first-party cookie such as a vendor's own ID cookie. Add the shared helpers the vendor providers reuse, and the operator page they document themselves on. + +**Architecture:** The JS SDK (TSJS) normalizes and hashes the visitor's email in the browser and writes the digest to `ts-hem`. On each request, `EcContext::decide_writer` reads `ts-hem` into a `HashedEmail` and copies every other non-reserved cookie into a map. Both travel on `RequestIdentity` to the coordinator's `ResolveSignals`, where `ResolveInput::hashed_email()` and the new `ResolveInput::cookie(name)` expose them only to providers whose descriptor declares them. `ts-hem` is a Trusted Server identity cookie: it is stripped from Prebid Server and origin requests, never stored in KV, and no provider can declare it as a raw cookie. Two small modules, `integrations/vendor_eid.rs` and `integrations/vendor_request.rs`, hold parsing and request helpers that more than one vendor uses. + +**Tech Stack:** Rust 1.95 (edition 2024), `trusted-server-core` (`hex`, `cookie`, `serde`, `error-stack`, all already dependencies), TypeScript in `crates/trusted-server-js/lib` (Vitest), VitePress docs. + +**Spec:** `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`. Section 4.4 defines `ResolveInput` and `HashedEmail`, section 9 the privacy rules, section 12 the vendor examples. Section 12.4 names "an input that reads a named first-party cookie" as an additive extension; this plan adds it for resolving providers only. The framework plan `docs/superpowers/plans/2026-10-22-server-side-identity-providers.md` built everything this plan extends. Writing guide for provider authors: `docs/guide/identity-providers.md`. + +**Required by:** every vendor plan dated 2026-10-28 (`2026-10-28-identity-provider-id5.md`, `-liveintent.md`, `-liveramp.md`, `-uid2.md`, `-connectid.md`, `-lockr-capture.md`). + +## Global Constraints + +- Follow `AGENTS.md`. Bare `cargo test` / `cargo build` fail at the workspace root; use the aliases (`cargo test-fastly`, `cargo clippy-fastly`). +- Errors use `error-stack` with `derive_more::Display` enums. `expect("should ...")`, never `unwrap()`. Tests follow Arrange-Act-Assert with descriptive assertion messages. +- No local imports inside functions. Comments on their own line above the code. Every public item has a doc comment; every builder method and getter is `#[must_use]`. +- Use only fictional data in code, tests and docs (`example.com` hosts, `person@example.com`). +- Never log a hashed email, a cookie value or a UID. +- The hashed email is never stored in KV, never forwarded to the origin, and never sent to Prebid Server. It reaches only providers that declare `Input::HashedEmail`. +- Cookie names compare exactly when stripping (no prefixes), and case-insensitively when deciding whether a name is reserved. +- Quick test command for a module: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib `. + +## File Structure + +| File | Change | Responsibility | +| -------------------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/constants.rs` | Modify | `COOKIE_TS_HEM` | +| `crates/trusted-server-core/src/cookies.rs` | Modify | Add `ts-hem` to both strip lists | +| `crates/trusted-server-core/src/ec/identity/input.rs` | Modify | `HashedEmail::from_cookie_hex`, `ResolveSignals::cookies`, `cookie()` | +| `crates/trusted-server-core/src/ec/identity/descriptor.rs` | Modify | `Resolves::with_cookie`, cookie-name checks in `build()`, `Access::Cookie` | +| `crates/trusted-server-core/src/ec/identity/test_support.rs` | Modify | Harness `with_cookie` | +| `crates/trusted-server-core/src/ec/identity/request.rs` | Modify | `RequestIdentity` and `WriterDecision` carry the HEM and cookies | +| `crates/trusted-server-core/src/ec/mod.rs` | Modify | `provider_cookie_signals`, called from `decide_writer` | +| `crates/trusted-server-core/src/ec/identity/flush.rs` | Modify | Pass the HEM and cookies into `ResolveSignals`; HEM availability | +| `crates/trusted-server-js/lib/src/core/hashed_email.ts` | Create | `setEmail`, `setHashedEmail`, `clearHashedEmail`, `normalizeEmail` | +| `crates/trusted-server-js/lib/test/core/hashed_email.test.ts` | Create | Tests for the above | +| `crates/trusted-server-js/lib/src/core/{index,types}.ts` | Modify | Expose the three functions on `tsjs` | +| `crates/trusted-server-core/src/integrations/vendor_eid.rs` | Create | Permissive parsing of vendor-returned complete EIDs | +| `crates/trusted-server-core/src/integrations/vendor_request.rs` | Create | `is_routable_ip` | +| `crates/trusted-server-core/src/integrations/mod.rs` | Modify | Declare the two helper modules | +| `docs/guide/integrations/identity-vendors.md` | Create | Operator page; each vendor plan adds its section | +| `docs/guide/identity-providers.md`, `docs/.vitepress/config.mts`, `CHANGELOG.md` | Modify | Author guide, sidebar, changelog | + +--- + +### Task 1: The `ts-hem` cookie name and its strips + +`ts-hem` must never leave the edge. Adding it to `TS_IDENTITY_COOKIE_NAMES` strips it from Prebid Server requests (and, in Task 3, makes it undeclarable as a provider cookie). Adding it to `TS_ORIGIN_STRIPPED_COOKIE_NAMES` strips it from publisher-origin requests. + +**Files:** + +- Modify: `crates/trusted-server-core/src/constants.rs` +- Modify: `crates/trusted-server-core/src/cookies.rs` (the two constants and the test `strip_cookie_pairs_removes_exact_names_only`) + +**Interfaces:** + +- Produces: `pub const COOKIE_TS_HEM: &str = "ts-hem";` in `crate::constants`. + +- [ ] **Step 1: Extend the strip test so it fails** + +In `cookies.rs`, replace the body of `strip_cookie_pairs_removes_exact_names_only`: + +```rust + #[test] + fn strip_cookie_pairs_removes_exact_names_only() { + let header = "ts-ec=abc; ts-eids=W10; __Host-ts-eids-s=eyJ2IjoxfQ; ts-hem=00ff; other=1; \ + ts-eids-extra=keep"; + + let stripped = strip_cookie_pairs(header, TS_ORIGIN_STRIPPED_COOKIE_NAMES); + + assert_eq!( + stripped, "ts-ec=abc; other=1; ts-eids-extra=keep", + "should drop the ID and hashed-email cookies and keep ts-ec and a cookie that only \ + shares a prefix" + ); + } +``` + +- [ ] **Step 2: Run it and see it fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib cookies::tests::strip_cookie_pairs_removes_exact_names_only` +Expected: FAIL, the stripped header still contains `ts-hem=00ff`. + +- [ ] **Step 3: Add the constant and the strips** + +In `constants.rs`, after `COOKIE_TS_EIDS_S`: + +```rust +/// Cookie written by the Trusted Server JS SDK holding the lowercase hex +/// SHA-256 of the visitor's normalized email address. Read per request as the +/// identity providers' hashed-email input; never stored in KV. +pub const COOKIE_TS_HEM: &str = "ts-hem"; +``` + +In `cookies.rs`, import it and extend both lists: + +```rust +use crate::constants::{ + COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_TS_EC, COOKIE_TS_EIDS, + COOKIE_TS_EIDS_S, COOKIE_TS_HEM, COOKIE_US_PRIVACY, +}; +``` + +```rust +pub const TS_IDENTITY_COOKIE_NAMES: &[&str] = &[ + COOKIE_TS_EC, + COOKIE_TS_EIDS, + COOKIE_TS_EIDS_S, + COOKIE_TS_HEM, +]; + +/// Identity cookies removed from publisher-origin requests. +/// +/// `ts-ec` still reaches the origin. The ID cookies +/// are removed so that they cannot push the origin's `Cookie` header past +/// common 8 KB header limits, and the hashed email never leaves the edge. +pub const TS_ORIGIN_STRIPPED_COOKIE_NAMES: &[&str] = + &[COOKIE_TS_EIDS, COOKIE_TS_EIDS_S, COOKIE_TS_HEM]; +``` + +- [ ] **Step 4: Run the cookie tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib cookies` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/constants.rs crates/trusted-server-core/src/cookies.rs +git commit -S --signoff -m "Strip the ts-hem hashed-email cookie from origin and Prebid Server requests" +``` + +--- + +### Task 2: Parse a `HashedEmail` from the cookie + +`HashedEmail` already exists (32 digest bytes, redacting `Debug`, `to_hex`, `to_base64`), but until now only tests could build one. The cookie holds 64 hex digits. A malformed value is treated as absent rather than as an error, so a bad cookie never fails a request. + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identity/input.rs` (`impl HashedEmail`, tests) + +**Interfaces:** + +- Produces: `pub(crate) fn HashedEmail::from_cookie_hex(value: &str) -> Option`. + +- [ ] **Step 1: Write the failing tests** + +Add to the `tests` module of `input.rs`: + +```rust + #[test] + fn hashed_email_parses_the_cookie_hex_in_either_case() { + let lower = HashedEmail::from_cookie_hex(&"ab".repeat(32)).expect("should parse lowercase"); + let upper = HashedEmail::from_cookie_hex(&"AB".repeat(32)).expect("should parse uppercase"); + + assert_eq!(lower, upper, "should read both cases as one digest"); + assert_eq!(lower.as_bytes(), &[0xab; 32], "should decode the digest"); + } + + #[test] + fn hashed_email_rejects_a_cookie_that_is_not_a_digest() { + for value in [ + "", + "ab", + &"ab".repeat(31), + &"ab".repeat(33), + &"zz".repeat(32), + ] { + assert!( + HashedEmail::from_cookie_hex(value).is_none(), + "should reject `{value}`" + ); + } + } +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib ec::identity::input::tests::hashed_email` +Expected: FAIL to compile, `no function or associated item named from_cookie_hex`. + +- [ ] **Step 3: Implement** + +In `impl HashedEmail`, before `from_bytes`, and update `from_bytes`'s doc comment: + +```rust + /// From the `ts-hem` cookie value: 64 hex digits of the digest, in + /// either case. `None` for anything else, so a malformed cookie is + /// treated as absent. + #[must_use] + pub(crate) fn from_cookie_hex(value: &str) -> Option { + let value = value.trim(); + if value.len() != 64 { + return None; + } + let mut bytes = [0u8; 32]; + hex::decode_to_slice(value, &mut bytes).ok()?; + Some(Self(bytes)) + } + + /// From digest bytes. Test support only: the one runtime source is the + /// `ts-hem` cookie, read with [`Self::from_cookie_hex`]. +``` + +Change the doc comment on `ResolveInput::hashed_email` from "A hashed email. Never available in v1." to "The hashed email from the `ts-hem` cookie.". In `descriptor.rs`, change the doc comment on `Input::HashedEmail` to: + +```rust + /// A SHA-256 of a normalized email address, from the `ts-hem` cookie the + /// JS SDK writes. Never stored. +``` + +- [ ] **Step 4: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib ec::identity::input` +Expected: PASS. (`from_cookie_hex` is unused outside tests until Task 4; a dead-code warning is expected until then.) + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity/input.rs crates/trusted-server-core/src/ec/identity/descriptor.rs +git commit -S --signoff -m "Parse a hashed email from the ts-hem cookie value" +``` + +--- + +### Task 3: Declared first-party cookie input + +LiveIntent needs its first-party `_lc2_fpi` cookie. A provider declares each cookie name with `Resolves::with_cookie(name)` and reads it with `input.cookie(name)?`. Like every other input, reading an undeclared name is a provider bug: it returns `UndeclaredInput` and is recorded even if the caller hides the error. A declared cookie is never required, so it does not affect eligibility. `build()` refuses names that are not RFC 6265 tokens, and Trusted Server's own identity cookies and the consent cookies, which providers read through `consent()` instead. + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identity/descriptor.rs` (`Resolves`, `ProviderDescriptorBuilder::build`, `Access`, tests) +- Modify: `crates/trusted-server-core/src/ec/identity/input.rs` (`ResolveSignals`, `ResolveInput::cookie`, the test `signals()` helper, tests) +- Modify: `crates/trusted-server-core/src/ec/identity/test_support.rs` (`HarnessSetup`, `ProviderHarness`, `signals()`) +- Modify: `crates/trusted-server-core/src/ec/identity/flush.rs` (`add_dispatch_results`) + +**Interfaces:** + +- Consumes: `TS_IDENTITY_COOKIE_NAMES` (now with `ts-hem`, Task 1), `CONSENT_COOKIE_NAMES` from `crate::cookies`. +- Produces: + - `pub fn Resolves::with_cookie(self, name: &str) -> Self` + - `pub(crate) fn Resolves::cookies(&self) -> &BTreeSet` + - `Access::Cookie` (displayed `cookie`) + - `pub(crate) cookies: BTreeMap` on `ResolveSignals` + - `pub fn ResolveInput::cookie(&self, name: &str) -> Result, Report>` + - `pub fn HarnessSetup::with_cookie(self, name: &str, value: &str) -> Self` and the same on `ProviderHarness` + +- [ ] **Step 1: Write the failing descriptor tests** + +In `descriptor.rs` tests, add: + +```rust + #[test] + fn keeps_declared_cookies() { + let descriptor = ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie("_vendor_fpid") + .with_cookie("_vendor_fpid"), + ) + .build() + .expect("should build"); + + let cookies: Vec<&str> = descriptor + .resolves() + .expect("should resolve") + .cookies() + .iter() + .map(String::as_str) + .collect(); + assert_eq!(cookies, ["_vendor_fpid"], "should keep each name once"); + } +``` + +and append these cases to the `cases` vector in `build_rejects_invalid_descriptors`: + +```rust + ( + "cookie name with a separator", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie("vendor;id"), + ), + ), + ( + "empty cookie name", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie(""), + ), + ), + ( + "Trusted Server identity cookie", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie("TS-HEM"), + ), + ), + ( + "consent cookie", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie("euconsent-v2"), + ), + ), +``` + +- [ ] **Step 2: Write the failing input test** + +In `input.rs` tests, add a cookie to the `signals()` helper's struct literal: + +```rust + cookies: BTreeMap::from([("_vendor_fpid".to_owned(), "fpid-value".to_owned())]), +``` + +and add: + +```rust + #[test] + fn declared_cookies_are_readable_and_undeclared_ones_are_recorded() { + let descriptor = descriptor( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_cookie("_vendor_fpid") + .with_cookie("_vendor_other"), + ); + let signals = signals(); + let input = ResolveInput::new( + &descriptor, + kinds(&descriptor, ResolveKind::Initial), + &signals, + ); + + assert_eq!( + input + .cookie("_vendor_fpid") + .expect("should read a declared cookie"), + Some("fpid-value"), + "should return the cookie value" + ); + assert_eq!( + input + .cookie("_vendor_other") + .expect("should read a declared cookie"), + None, + "should return None for a declared cookie the request lacks" + ); + let error = input + .cookie("_undeclared") + .expect_err("should refuse an undeclared cookie"); + assert!( + matches!( + error.current_context(), + IdentityProviderError::UndeclaredInput { + access: Access::Cookie + } + ), + "should report UndeclaredInput" + ); + assert_eq!( + input.recorded_errors(), + [Access::Cookie], + "should record the undeclared read" + ); + } +``` + +- [ ] **Step 3: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib ec::identity` +Expected: FAIL to compile (`with_cookie`, `cookies`, `Access::Cookie` do not exist). + +- [ ] **Step 4: Implement the descriptor side** + +In `descriptor.rs`, import the reserved names: + +```rust +use crate::cookies::{CONSENT_COOKIE_NAMES, TS_IDENTITY_COOKIE_NAMES}; +``` + +Add `cookies: BTreeSet` to `struct Resolves`, initialise it with `BTreeSet::new()` in `Resolves::new`, and add the builder method and getter: + +```rust + /// A first-party cookie the provider may read, such as the vendor's own + /// ID cookie on the publisher's domain. Repeatable. It is passed when the + /// request carries it and is never required. `build()` refuses Trusted + /// Server's identity cookies and the consent cookies, which providers + /// read through [`ResolveInput::consent`](super::input::ResolveInput::consent). + #[must_use] + pub fn with_cookie(mut self, name: &str) -> Self { + self.cookies.insert(name.to_owned()); + self + } +``` + +```rust + /// The first-party cookies the provider may read. + #[must_use] + pub(crate) fn cookies(&self) -> &BTreeSet { + &self.cookies + } +``` + +In `ProviderDescriptorBuilder::build`, inside the block that checks `resolves` (after the endpoint checks), add: + +```rust + for name in &resolves.cookies { + if !is_cookie_name(name) { + return Err(fail(&format!("cookie `{name}` is not a valid cookie name"))); + } + if is_reserved_cookie(name) { + return Err(fail(&format!( + "cookie `{name}` is a Trusted Server identity or consent cookie" + ))); + } + } +``` + +and after `impl ProviderDescriptorBuilder`: + +```rust +/// Whether `name` is an RFC 6265 cookie-name token. +fn is_cookie_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|byte| byte.is_ascii_graphic() && !b"()<>@,;:\\\"/[]?={}".contains(&byte)) +} + +/// Trusted Server's own identity cookies and the consent cookies, which no +/// provider reads directly. +fn is_reserved_cookie(name: &str) -> bool { + TS_IDENTITY_COOKIE_NAMES + .iter() + .chain(CONSENT_COOKIE_NAMES) + .any(|reserved| reserved.eq_ignore_ascii_case(name)) +} +``` + +Add the variant to `enum Access`: + +```rust + /// A first-party cookie, without `with_cookie()` for its name. + #[display("cookie")] + Cookie, +``` + +- [ ] **Step 5: Implement the input side** + +In `input.rs`, add the field to `ResolveSignals`: + +```rust + /// The request's first-party cookies other than Trusted Server's own and + /// the consent cookies. A provider reads only the names it declared. + pub(crate) cookies: BTreeMap, +``` + +and the accessor on `ResolveInput`, after `hashed_email`: + +```rust + /// The value of the first-party cookie `name`, or `None` when the request + /// does not carry it. + /// + /// # Errors + /// + /// `UndeclaredInput { access: Cookie }` when the descriptor did not + /// declare `name` with + /// [`Resolves::with_cookie`](super::descriptor::Resolves::with_cookie). + pub fn cookie(&self, name: &str) -> Result, Report> { + if !self + .descriptor + .resolves() + .is_some_and(|resolves| resolves.cookies().contains(name)) + { + return Err(self.record(Access::Cookie)); + } + Ok(self.signals.cookies.get(name).map(String::as_str)) + } +``` + +- [ ] **Step 6: Give the harness and the coordinator the field** + +In `test_support.rs`, add `cookies: BTreeMap` to `HarnessSetup`, and: + +```rust + /// Adds a first-party cookie the request carries. Repeatable. + #[must_use] + pub fn with_cookie(mut self, name: &str, value: &str) -> Self { + self.cookies.insert(name.to_owned(), value.to_owned()); + self + } +``` + +On `ProviderHarness`, next to `with_hashed_email`: + +```rust + /// Adds a first-party cookie the request carries. Repeatable. + #[must_use] + pub fn with_cookie(self, name: &str, value: &str) -> Self { + self.map_setup(|setup| setup.with_cookie(name, value)) + } +``` + +and in `ProviderHarness::signals`, add `cookies: self.setup.cookies.clone(),` to the `ResolveSignals` literal. + +In `flush.rs` `add_dispatch_results`, add `cookies: BTreeMap::new(),` to the `ResolveSignals` literal for now. Task 4 replaces it with the request's cookies. + +- [ ] **Step 7: Run the identity tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib ec::identity` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity/ +git commit -S --signoff -m "Let identity providers declare and read first-party cookies" +``` + +--- + +### Task 4: Carry the hashed email and cookies from the request + +The writer decision is where `EcContext` already captures the per-request provider signals (user agent, `DNT`, page URL). This task captures `ts-hem` and the provider-readable cookies there too, carries them on `RequestIdentity`, and hands them to `ResolveSignals`. `request_facts` reports `Input::HashedEmail` as available when a valid `ts-hem` was present, so a provider that needs it becomes due only for signed-in visitors. The cookie map excludes Trusted Server's own cookies and the consent cookies, matched case-insensitively, the same set `build()` refuses. A malformed `Cookie` header yields neither signal. + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identity/request.rs` (`RequestIdentity`, `WriterDecision`, `set_writer_decision`) +- Modify: `crates/trusted-server-core/src/ec/mod.rs` (`provider_cookie_signals`, `decide_writer`, the test-only `WriterDecision` literal, tests) +- Modify: `crates/trusted-server-core/src/ec/identity/flush.rs` (`request_facts`, `add_dispatch_results`, test literals) +- Modify: every other `WriterDecision { .. }` literal: `auction/endpoints.rs`, `ec/finalize.rs`, `publisher.rs`, `ec/identity/test_support.rs` (`hold_fixture`, `capture_fixture`) + +**Interfaces:** + +- Consumes: `COOKIE_TS_HEM` (Task 1), `HashedEmail::from_cookie_hex` (Task 2), `ResolveSignals::cookies` (Task 3). +- Produces: `RequestIdentity::hashed_email() -> Option<&HashedEmail>`, `RequestIdentity::cookies() -> &BTreeMap`, `WriterDecision { hashed_email: Option, cookies: BTreeMap, .. }`. + +- [ ] **Step 1: Write the failing test** + +In `ec/mod.rs` tests, add: + +```rust + #[test] + fn decide_writer_captures_the_hashed_email_and_provider_cookies() { + let request = Request::builder() + .uri("https://www.example.com/a") + .header("sec-fetch-dest", "document") + .header( + header::COOKIE, + format!( + "ts-hem={}; _vendor_fpid=fpid-value; ts-ec=ignored; euconsent-v2=ignored", + "AB".repeat(32) + ), + ) + .body(EdgeBody::empty()) + .expect("should build the request"); + let mut context = EcContext::default(); + + context.decide_writer(&request, true, true); + + assert_eq!( + context.identity().hashed_email().map(HashedEmail::as_bytes), + Some(&[0xab; 32]), + "should read the hashed email from ts-hem" + ); + let names: Vec<&str> = context + .identity() + .cookies() + .keys() + .map(String::as_str) + .collect(); + assert_eq!( + names, + ["_vendor_fpid"], + "should keep only cookies that are not Trusted Server or consent cookies" + ); + } + + #[test] + fn decide_writer_treats_a_malformed_hashed_email_as_absent() { + let request = Request::builder() + .uri("https://www.example.com/a") + .header(header::COOKIE, "ts-hem=person@example.com") + .body(EdgeBody::empty()) + .expect("should build the request"); + let mut context = EcContext::default(); + + context.decide_writer(&request, true, true); + + assert!( + context.identity().hashed_email().is_none(), + "should ignore a ts-hem value that is not a digest" + ); + } +``` + +- [ ] **Step 2: Run it and see it fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib ec::tests::decide_writer` +Expected: FAIL to compile (`hashed_email` and `cookies` are not methods of `RequestIdentity`). + +- [ ] **Step 3: Carry the fields on `RequestIdentity`** + +In `request.rs`, import `super::input::HashedEmail`, add `hashed_email: Option` and `cookies: BTreeMap` to `RequestIdentity` and to `WriterDecision`, assign them in `set_writer_decision`: + +```rust + self.hashed_email = decision.hashed_email; + self.cookies = decision.cookies; +``` + +and add the getters: + +```rust + /// The hashed email from the request's `ts-hem` cookie. + #[must_use] + pub(crate) fn hashed_email(&self) -> Option<&HashedEmail> { + self.hashed_email.as_ref() + } + + /// The request's first-party cookies other than Trusted Server's own and + /// the consent cookies, for providers that declare them. + #[must_use] + pub(crate) fn cookies(&self) -> &BTreeMap { + &self.cookies + } +``` + +- [ ] **Step 4: Read them in `decide_writer`** + +In `ec/mod.rs`, update the imports: + +```rust +use std::collections::BTreeMap; + +use crate::constants::{COOKIE_TS_EC, COOKIE_TS_HEM}; +use crate::cookies::{CONSENT_COOKIE_NAMES, TS_IDENTITY_COOKIE_NAMES, handle_request_cookies}; + +use self::identity::input::HashedEmail; +``` + +Add, after `parse_ec_from_request`: + +```rust +/// The hashed email from the `ts-hem` cookie and the cookies identity +/// providers may read: every first-party cookie except Trusted Server's own +/// and the consent cookies. A malformed `Cookie` header gives neither. +fn provider_cookie_signals( + req: &Request, +) -> (Option, BTreeMap) { + let Some(jar) = handle_request_cookies(req).ok().flatten() else { + return (None, BTreeMap::new()); + }; + let hashed_email = jar + .get(COOKIE_TS_HEM) + .and_then(|cookie| HashedEmail::from_cookie_hex(cookie.value())); + let cookies = jar + .iter() + .filter(|cookie| { + !TS_IDENTITY_COOKIE_NAMES + .iter() + .chain(CONSENT_COOKIE_NAMES) + .any(|reserved| reserved.eq_ignore_ascii_case(cookie.name())) + }) + .map(|cookie| (cookie.name().to_owned(), cookie.value().to_owned())) + .collect(); + (hashed_email, cookies) +} +``` + +In `decide_writer`, before `set_writer_decision`, call it and pass the results: + +```rust + let (hashed_email, cookies) = provider_cookie_signals(req); + self.identity.set_writer_decision(WriterDecision { + writer, + is_real_browser, + user_agent, + dnt, + page_url, + hashed_email, + cookies, + }); +``` + +Extend `decide_writer`'s doc comment: "Also captures the user agent, `DNT`, the page URL without query or fragment, the hashed email from the `ts-hem` cookie, and the first-party cookies identity providers may declare." In the test-only constructor at the bottom of `impl EcContext` that builds a `WriterDecision`, add `hashed_email: None, cookies: BTreeMap::new(),`. + +- [ ] **Step 5: Hand them to providers** + +In `flush.rs` `request_facts`, add a row to the availability table: + +```rust + (Input::HashedEmail, identity.hashed_email().is_some()), +``` + +In `add_dispatch_results`, replace `hashed_email: None,` with `hashed_email: identity.hashed_email().cloned(),` and the Task 3 placeholder `cookies: BTreeMap::new(),` with `cookies: identity.cookies().clone(),`. + +- [ ] **Step 6: Fix the remaining `WriterDecision` literals** + +Add these two lines to every remaining `WriterDecision { .. }` literal (compile errors list them: tests in `auction/endpoints.rs`, `ec/finalize.rs`, `ec/identity/flush.rs`, `publisher.rs`, and `hold_fixture` and `capture_fixture` in `ec/identity/test_support.rs`): + +```rust + hashed_email: None, + cookies: BTreeMap::new(), +``` + +Use `std::collections::BTreeMap::new()` where the module does not import `BTreeMap`. + +- [ ] **Step 7: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib ec` +Expected: PASS. + +Run: `cargo test-fastly` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add crates/trusted-server-core/src/ +git commit -S --signoff -m "Read the hashed email and provider cookies on the writer request" +``` + +--- + +### Task 5: TSJS functions that write `ts-hem` + +The page owns the email address; the edge only ever sees its digest. `setEmail` normalizes the address the way UID2 and most ID vendors expect (trim, lowercase, and for Gmail drop dots and any `+suffix` from the local part), hashes it with WebCrypto, and stores the lowercase hex digest. `setHashedEmail` stores a digest the page already has. `clearHashedEmail` removes it on sign-out. The cookie is `Path=/; Secure; SameSite=Lax` with a 30-day `Max-Age`, not `HttpOnly`, since the page writes it. + +**Files:** + +- Create: `crates/trusted-server-js/lib/src/core/hashed_email.ts` +- Create: `crates/trusted-server-js/lib/test/core/hashed_email.test.ts` +- Modify: `crates/trusted-server-js/lib/src/core/index.ts`, `crates/trusted-server-js/lib/src/core/types.ts` + +**Interfaces:** + +- Produces: `normalizeEmail(email: string): string | undefined`, `setEmail(email: string): Promise`, `setHashedEmail(hex: string): boolean`, `clearHashedEmail(): void`, and the same three (not `normalizeEmail`) on `window.tsjs`. + +- [ ] **Step 1: Write the failing tests** + +Create `test/core/hashed_email.test.ts`: + +```ts +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + clearHashedEmail, + normalizeEmail, + setEmail, + setHashedEmail, +} from '../../src/core/hashed_email' + +const EXAMPLE_HASH = 'ab'.repeat(32) + +describe('core/hashed_email', () => { + let writes: string[] + + beforeEach(() => { + writes = [] + Object.defineProperty(document, 'cookie', { + configurable: true, + get: () => '', + set: (assignment: string) => { + writes.push(assignment) + }, + }) + }) + + afterEach(() => { + delete (document as unknown as { cookie?: string }).cookie + }) + + it('normalizes case, whitespace and Gmail aliases', () => { + expect(normalizeEmail(' Person@Example.COM ')).toBe('person@example.com') + expect(normalizeEmail('first.last+news@gmail.com')).toBe( + 'firstlast@gmail.com' + ) + expect(normalizeEmail('first.last+news@example.com')).toBe( + 'first.last+news@example.com' + ) + expect(normalizeEmail('not-an-address')).toBeUndefined() + expect(normalizeEmail('@example.com')).toBeUndefined() + }) + + it('stores a hex digest in ts-hem', () => { + expect(setHashedEmail(EXAMPLE_HASH.toUpperCase())).toBe(true) + expect(writes).toEqual([ + `ts-hem=${EXAMPLE_HASH}; Path=/; Secure; SameSite=Lax; Max-Age=2592000`, + ]) + }) + + it('refuses a value that is not a digest', () => { + expect(setHashedEmail('person@example.com')).toBe(false) + expect(writes).toEqual([]) + }) + + it('hashes the normalized email without storing the address', async () => { + const stored = await setEmail(' Person@Example.com ') + + const expected = Array.from( + new Uint8Array( + await crypto.subtle.digest( + 'SHA-256', + new TextEncoder().encode('person@example.com') + ) + ), + (byte) => byte.toString(16).padStart(2, '0') + ).join('') + expect(stored).toBe(true) + expect(writes).toEqual([ + `ts-hem=${expected}; Path=/; Secure; SameSite=Lax; Max-Age=2592000`, + ]) + expect(writes[0]).not.toContain('example.com') + }) + + it('clears ts-hem', () => { + clearHashedEmail() + expect(writes).toEqual(['ts-hem=; Path=/; Secure; SameSite=Lax; Max-Age=0']) + }) +}) +``` + +- [ ] **Step 2: Run it and see it fail** + +Run (with the Node 24 pinned in `.tool-versions`, for example through `mise exec --`): `cd crates/trusted-server-js/lib && npx vitest run test/core/hashed_email.test.ts` +Expected: FAIL, cannot resolve `../../src/core/hashed_email`. + +- [ ] **Step 3: Implement** + +Create `src/core/hashed_email.ts`: + +```ts +// Writes the first-party `ts-hem` cookie: the lowercase hex SHA-256 of the +// visitor's normalized email address. TS reads it per request as the identity +// providers' hashed-email input and never stores it, forwards it to the +// origin, or sends it to Prebid Server. + +const HEM_COOKIE_NAME = 'ts-hem' + +const ATTRIBUTES = 'Path=/; Secure; SameSite=Lax' + +// Thirty days: the page sets it again on each sign-in. +const HEM_COOKIE_MAX_AGE = 30 * 24 * 60 * 60 + +const SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/ + +const GMAIL_DOMAINS = new Set(['gmail.com', 'googlemail.com']) + +/** + * Normalizes an email address the way UID2 and most ID vendors hash it: + * trims and lowercases, and for Gmail drops dots and any `+suffix` from the + * local part. + * + * @param email the address the visitor gave + * @returns the normalized address, or `undefined` when it is not an address + */ +export function normalizeEmail(email: string): string | undefined { + const trimmed = email.trim().toLowerCase() + const at = trimmed.indexOf('@') + // One `@` only: `a@b@example.com` is not an address. + if ( + at <= 0 || + at === trimmed.length - 1 || + trimmed.includes('@', at + 1) || + /\s/.test(trimmed) + ) { + return undefined + } + let local = trimmed.slice(0, at) + const domain = trimmed.slice(at + 1) + if (GMAIL_DOMAINS.has(domain)) { + local = local.split('+')[0].replace(/\./g, '') + if (local.length === 0) { + return undefined + } + } + return `${local}@${domain}` +} + +function writeCookie(value: string, maxAge: number): void { + try { + document.cookie = `${HEM_COOKIE_NAME}=${value}; ${ATTRIBUTES}; Max-Age=${maxAge}` + } catch { + // Cookie access can throw in sandboxed frames; the hashed email is optional. + } +} + +/** + * Stores an already-hashed email. + * + * @param hex the SHA-256 of the normalized address, as 64 hex digits + * @returns whether the value was a digest and was stored + */ +export function setHashedEmail(hex: string): boolean { + const value = hex.trim().toLowerCase() + if (!SHA256_HEX_PATTERN.test(value)) { + return false + } + writeCookie(value, HEM_COOKIE_MAX_AGE) + return true +} + +/** + * Normalizes and hashes an email address in the browser, then stores the + * digest. The address itself is never stored or sent. + * + * @param email the address the visitor gave + * @returns whether a digest was stored + */ +export async function setEmail(email: string): Promise { + const normalized = normalizeEmail(email) + const subtle = globalThis.crypto?.subtle + if (normalized === undefined || subtle === undefined) { + return false + } + const digest = await subtle.digest( + 'SHA-256', + new TextEncoder().encode(normalized) + ) + const hex = Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, '0') + ).join('') + return setHashedEmail(hex) +} + +/** Removes the stored hashed email, for example on sign-out. */ +export function clearHashedEmail(): void { + writeCookie('', 0) +} +``` + +- [ ] **Step 4: Expose them on `tsjs`** + +In `src/core/types.ts`, add to `interface TsjsApi` after `requestAds`: + +```ts + /** + * Normalizes and hashes the visitor's email in the browser and stores the + * SHA-256 in the first-party `ts-hem` cookie, the identity providers' + * hashed-email input. Resolves to whether a digest was stored. + */ + setEmail?(email: string): Promise; + /** Stores an already-computed SHA-256 (64 hex digits) in `ts-hem`. */ + setHashedEmail?(hex: string): boolean; + /** Removes `ts-hem`, for example on sign-out. */ + clearHashedEmail?(): void; +``` + +In `src/core/index.ts`, import them and attach them after `api.requestAds = requestAds;`: + +```ts +import { clearHashedEmail, setEmail, setHashedEmail } from './hashed_email' +``` + +```ts +api.setEmail = setEmail +api.setHashedEmail = setHashedEmail +api.clearHashedEmail = clearHashedEmail +``` + +and add `'setEmail'`, `'setHashedEmail'`, `'clearHashedEmail'` to the `methods` list in the `tsjs initialized` log call. + +- [ ] **Step 5: Run the JS checks** + +Run: `cd crates/trusted-server-js/lib && npx vitest run && npm run lint && npm run format` +Expected: PASS, and `format` leaves no diff. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-js/lib/src/core/ crates/trusted-server-js/lib/test/core/hashed_email.test.ts +git commit -S --signoff -m "Add tsjs functions that store a hashed email in ts-hem" +``` + +--- + +### Task 6: Shared vendor helpers + +Two helpers are used by more than one vendor plan, so they land here once. + +- `VendorEid`: vendors such as ID5 and Lockr return complete `OpenRTB` EIDs. `crate::openrtb::Eid` uses `deny_unknown_fields`, which is right for Trusted Server's own output but would fail a whole vendor response the day the vendor adds a field. `VendorEid` accepts unknown fields and turns one EID into a `ResolvedEid` under a declared source, keeping every non-empty UID with its `atype` and `ext`, and the EID-level `inserter`, `matcher`, `mm` and `ext`. +- `is_routable_ip`: vendors that match on the client IP answer "no match" for a loopback, private, link-local or carrier-grade NAT address (LiveIntent answers 204 for a loopback `pip`, which is what a local dev server sends). Leaving such an address out lets the vendor match on its other signals. + +**Files:** + +- Create: `crates/trusted-server-core/src/integrations/vendor_eid.rs` +- Create: `crates/trusted-server-core/src/integrations/vendor_request.rs` +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` + +**Interfaces:** + +- Produces: + - `pub(crate) struct VendorEid { pub source: String, pub uids: Vec, pub inserter: Option, pub matcher: Option, pub mm: Option, pub ext: Option }` (`Deserialize`) + - `pub(crate) fn VendorEid::to_resolved(&self, source: &SourceDomain) -> Option` + - `pub(crate) fn is_routable_ip(ip: IpAddr) -> bool` + +- [ ] **Step 1: Declare the modules** + +In `integrations/mod.rs`, next to the other module declarations: + +```rust +mod vendor_eid; +mod vendor_request; +``` + +- [ ] **Step 2: Write the failing tests** + +Create `vendor_eid.rs` with only its tests module for now: + +```rust +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn keeps_every_uid_and_eid_field() { + let eid: VendorEid = serde_json::from_value(json!({ + "source": "ids.example.com", + "uids": [ + { "id": "first", "atype": 1, "ext": { "linkType": 2 } }, + { "id": "" }, + { "id": "second", "future": true } + ], + "inserter": "inserter.example.com", + "matcher": "matcher.example.com", + "mm": 2, + "ext": { "segments": ["s1"] }, + "future": "ignored" + })) + .expect("should parse a vendor EID with unknown fields"); + let source: SourceDomain = "ids.example.com".parse().expect("should parse the source"); + + let resolved = eid.to_resolved(&source).expect("should resolve"); + + let ids: Vec<&str> = resolved.uids().iter().map(ResolvedUid::id).collect(); + assert_eq!( + ids, + ["first", "second"], + "should keep non-empty UIDs in order" + ); + assert_eq!(resolved.uids()[0].atype(), Some(1), "should keep the atype"); + assert_eq!( + resolved.inserter(), + Some("inserter.example.com"), + "should keep inserter" + ); + assert_eq!( + resolved.matcher(), + Some("matcher.example.com"), + "should keep matcher" + ); + assert_eq!(resolved.mm(), Some(2), "should keep mm"); + assert_eq!( + resolved.ext(), + Some(&json!({ "segments": ["s1"] })), + "should keep the EID ext" + ); + } + + #[test] + fn an_eid_without_ids_resolves_to_none() { + let eid: VendorEid = + serde_json::from_value(json!({ "source": "ids.example.com", "uids": [{ "id": "" }] })) + .expect("should parse"); + let source: SourceDomain = "ids.example.com".parse().expect("should parse the source"); + + assert!( + eid.to_resolved(&source).is_none(), + "should skip an EID with no ID" + ); + } +} +``` + +Create `vendor_request.rs` with only its tests module: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_public_addresses_and_drops_local_ones() { + for ip in ["192.0.2.10", "198.51.100.7", "2001:db8::1"] { + assert!( + is_routable_ip(ip.parse().expect("should parse")), + "should keep {ip}" + ); + } + for ip in [ + "127.0.0.1", + "0.0.0.0", + "10.1.2.3", + "172.16.0.1", + "192.168.1.1", + "169.254.1.1", + "100.64.0.1", + "::1", + "::", + "fd00::1", + "fe80::1", + "::ffff:127.0.0.1", + ] { + assert!( + !is_routable_ip(ip.parse().expect("should parse")), + "should drop {ip}" + ); + } + } +} +``` + +- [ ] **Step 3: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::vendor_` +Expected: FAIL to compile (`VendorEid`, `is_routable_ip` not found). + +- [ ] **Step 4: Implement `vendor_eid.rs`** + +Prepend to `vendor_eid.rs`: + +```rust +//! Permissive parsing of complete EIDs that identity vendors return. +//! +//! [`crate::openrtb::Eid`] denies unknown fields, so it suits Trusted +//! Server's own output, not a vendor's input: one new vendor field would fail +//! the whole response. + +use serde::Deserialize; + +use crate::ec::identity::{ResolvedEid, ResolvedUid, SourceDomain}; + +/// One EID as a vendor returns it. +#[derive(Debug, Deserialize)] +pub(crate) struct VendorEid { + pub source: String, + #[serde(default)] + pub uids: Vec, + pub inserter: Option, + pub matcher: Option, + pub mm: Option, + pub ext: Option, +} + +/// One UID of a [`VendorEid`]. +#[derive(Debug, Deserialize)] +pub(crate) struct VendorUid { + pub id: String, + pub atype: Option, + pub ext: Option, +} + +impl VendorEid { + /// The EID under `source`, with every UID and EID-level field the vendor + /// sent. `None` when it holds no UID with an ID. + pub(crate) fn to_resolved(&self, source: &SourceDomain) -> Option { + let mut uids = self + .uids + .iter() + .filter(|uid| !uid.id.is_empty()) + .map(VendorUid::to_resolved); + let mut eid = ResolvedEid::new(source, uids.next()?); + for uid in uids { + eid = eid.with_uid(uid); + } + if let Some(inserter) = &self.inserter { + eid = eid.with_inserter(inserter); + } + if let Some(matcher) = &self.matcher { + eid = eid.with_matcher(matcher); + } + if let Some(mm) = self.mm { + eid = eid.with_mm(mm); + } + if let Some(ext) = &self.ext { + eid = eid.with_ext(ext.clone()); + } + Some(eid) + } +} + +impl VendorUid { + fn to_resolved(&self) -> ResolvedUid { + let mut uid = ResolvedUid::new(self.id.clone()); + if let Some(atype) = self.atype { + uid = uid.with_atype(atype); + } + if let Some(ext) = &self.ext { + uid = uid.with_ext(ext.clone()); + } + uid + } +} +``` + +- [ ] **Step 5: Implement `vendor_request.rs`** + +Prepend to `vendor_request.rs`: + +```rust +//! Helpers shared by the identity vendors' requests. + +use std::net::IpAddr; + +/// Whether `ip` can identify a visitor to a vendor: not loopback, +/// unspecified, private, link-local or carrier-grade NAT. +/// +/// A vendor that matches on the client IP answers no match for such an +/// address, as it does for a request from a local dev server, so the IP is +/// left out instead. +pub(crate) fn is_routable_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + let [first, second, ..] = v4.octets(); + let carrier_grade_nat = first == 100 && (second & 0xc0) == 64; + !(v4.is_loopback() + || v4.is_unspecified() + || v4.is_private() + || v4.is_link_local() + || v4.is_broadcast() + || carrier_grade_nat) + } + IpAddr::V6(v6) => { + if let Some(v4) = v6.to_ipv4_mapped() { + return is_routable_ip(IpAddr::V4(v4)); + } + let first = v6.segments()[0]; + let unique_local = (first & 0xfe00) == 0xfc00; + let link_local = (first & 0xffc0) == 0xfe80; + !(v6.is_loopback() || v6.is_unspecified() || unique_local || link_local) + } + } +} +``` + +- [ ] **Step 6: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::vendor_` +Expected: PASS. Both modules are dead code in the library build until the first vendor plan uses them, so `cargo clippy-fastly` reports `dead_code` until then. Land this task together with the first vendor plan's PR, or run clippy after that plan's first task. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/ +git commit -S --signoff -m "Add shared vendor EID parsing and routable-IP helpers" +``` + +--- + +### Task 7: Operator and author documentation + +**Files:** + +- Create: `docs/guide/integrations/identity-vendors.md` +- Modify: `docs/.vitepress/config.mts`, `docs/guide/identity-providers.md`, `CHANGELOG.md` + +- [ ] **Step 1: Create the operator page** + +Create `docs/guide/integrations/identity-vendors.md`. Each vendor plan adds one row to the table and one `##` section after "Hashed email": + +````markdown +# Identity Vendors + +Trusted Server ships server-side [identity providers](/guide/identity-providers) for several vendors. Each one is an integration under `[integrations.]` that, once enabled, resolves or captures IDs at the edge and writes them to the Edge Cookie (EC) identity graph in KV. You don't need the vendor's browser module for this. + +| Integration | Vendor | Kind | Sources | Needs | +| ----------- | ------ | ---- | ------- | ----- | + +Criteo and The Trade Desk are not included. Their IDs live in the vendor's own third-party cookie, which Trusted Server cannot read at the publisher's edge. Browser ingest still carries those IDs from `ts-eids` when they are registered partners. + +## Before you enable one + +Every source a provider writes needs an `[[ec.partners]]` entry, or `ts config validate` refuses the config. When a provider declares more sources than you want, narrow it: + +```toml +[[ec.partners]] +name = "Example ID" +source_domain = "ids.example.com" +openrtb_atype = 1 +bidstream_enabled = true + +[ec.identity.providers.] +sources = ["ids.example.com"] +``` + +Provider timing, rate limits and ownership use the keys in the [Configuration Reference](/guide/configuration#identity-providers-ec-identity). + +## Hashed email + +Some providers need a hashed email, and others send one when it is present. The page supplies it after the visitor signs in. The JS SDK normalizes and hashes the address in the browser and stores only the SHA-256 in the first-party `ts-hem` cookie: + +```js +window.tsjs.que.push(async () => { + await window.tsjs.setEmail('person@example.com') +}) + +// Or, when the page already has the digest: +window.tsjs.setHashedEmail('<64 hex digits>') + +// On sign-out: +window.tsjs.clearHashedEmail() +``` + +Normalization trims and lowercases the address. For Gmail it also drops dots and any `+suffix` from the local part. The cookie lasts 30 days. + +Trusted Server reads `ts-hem` on each request and passes it only to providers that declare the hashed-email input. It never stores the digest in KV, and it strips the cookie before forwarding a request to the origin or to Prebid Server. A hashed email is personal data: set it only under the legal basis your consent platform records. +```` + +- [ ] **Step 2: Add the sidebar entry** + +In `docs/.vitepress/config.mts`, in the integrations list, before `{ text: 'Lockr', ... }`: + +```ts + { + text: 'Identity Vendors', + link: '/guide/integrations/identity-vendors', + }, +``` + +- [ ] **Step 3: Update the author guide** + +In `docs/guide/identity-providers.md`, replace "No vendor ships with the framework. Vendors arrive in their own specs." with: + +```markdown +The vendor providers that ship with Trusted Server, listed in [Identity Vendors](/guide/integrations/identity-vendors), are worked examples of the same API. +``` + +After the bullet list describing `UndeclaredInput` and `InputUnavailable`, add: + +```markdown +The hashed email (`Input::HashedEmail`) comes from the `ts-hem` cookie the page sets through the JS SDK, so it is present only for visitors who signed in. Treat it as wanted unless the vendor cannot work without it. + +A provider can also read a first-party cookie on the publisher's domain, such as the vendor's own ID cookie. Declare each name with `Resolves::with_cookie(name)` and read it with `input.cookie(name)?`, which returns `None` when the request lacks it. Trusted Server's own cookies and the consent cookies cannot be declared. Consent comes from `consent()`. +``` + +- [ ] **Step 4: Update the changelog** + +Under `### Added` in `CHANGELOG.md`, add: + +```markdown +- `tsjs.setEmail`, `tsjs.setHashedEmail` and `tsjs.clearHashedEmail` store the SHA-256 of the visitor's normalized email in the first-party `ts-hem` cookie, the identity providers' hashed-email input. Trusted Server reads it per request, never stores it, and strips it from requests to the origin and to Prebid Server. +- Identity providers can declare and read first-party cookies with `Resolves::with_cookie` and `ResolveInput::cookie`. +``` + +- [ ] **Step 5: Format and run the gates** + +Run: `cd docs && npm run format` +Run: `docs/node_modules/.bin/prettier --config docs/.prettierrc --check CHANGELOG.md` +Run: `cargo fmt --all -- --check && cargo test-fastly && cargo clippy-fastly` +Expected: PASS (see Task 6, Step 6, for the expected `dead_code` until the first vendor lands). + +- [ ] **Step 6: Commit** + +```bash +git add docs/ CHANGELOG.md +git commit -S --signoff -m "Document the hashed-email input and declared provider cookies" +``` + +## Self-Review Notes + +- `ts-hem` is in `TS_IDENTITY_COOKIE_NAMES` (Prebid Server strip and the reserved-cookie check) and `TS_ORIGIN_STRIPPED_COOKIE_NAMES` (origin strip): Task 1. +- Malformed `ts-hem` is absent, never an error: Tasks 2 and 4. +- `HashedEmail` availability drives eligibility through `request_facts`: Task 4, Step 5. +- A declared cookie is never required and never affects eligibility; an undeclared read is recorded: Task 3. diff --git a/docs/superpowers/plans/2026-10-28-identity-provider-liveintent.md b/docs/superpowers/plans/2026-10-28-identity-provider-liveintent.md new file mode 100644 index 0000000000..cc40d4ad6a --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-identity-provider-liveintent.md @@ -0,0 +1,805 @@ +# LiveIntent Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve LiveIntent's IDs at the edge with a server-side `IdentityProvider` under `[integrations.liveintent]`, one `OpenRTB` source per attribute the operator configures. + +**Architecture:** `integrations/liveintent.rs` holds `LiveIntentConfig`, `LiveIntentProvider` and `register`. The provider sends one `GET` to LiveIntent's identity exchange with the visitor's first-party `_lc2_fpi` cookie, consent, page origin and client IP, and one `resolve` parameter per configured attribute. A fixed table maps each attribute to the source it carries; only the configured attributes' sources are declared, so only those need `[[ec.partners]]` entries. The reply is a flat JSON object keyed by attribute name. + +**Tech Stack:** Rust 1.95 (edition 2024), `trusted-server-core`, `serde_json`, `base64`, `chrono`, `http`, `validator`, `error-stack` (all already dependencies). + +**Spec:** `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`, sections 4.2 to 4.6 (the provider API) and section 12 (vendor examples; LiveIntent follows the same shape). Operator page: `docs/guide/integrations/identity-vendors.md`. + +**Prerequisite:** `docs/superpowers/plans/2026-10-28-identity-provider-inputs.md`. This plan uses the declared first-party cookie input (`Resolves::with_cookie`, `ResolveInput::cookie`, harness `with_cookie`) and `is_routable_ip` (`integrations/vendor_request.rs`). + +## Why the request looks the way it does + +These were checked against LiveIntent's live exchange: + +- **`GET https://idx.liadm.com/idex//`.** The path names the distributor (or `prebid` when there is none) and the publisher (or `any`). +- **`Origin` header is required.** Without it LiveIntent answers 400 "Request origin could not be determined". The provider sends `Origin: https://`. +- **`did` is the distributor ID, and only when configured.** Any other `did` value gets 400 "Malformed did", so the provider never invents one. +- **`duid` is the first-party `_lc2_fpi` cookie** on the publisher's domain, which is why this provider needs the declared-cookie input. It is optional; LiveIntent still matches on other signals. +- **`pip` / `pip6` must be routable.** A loopback or private `pip` makes LiveIntent answer 204, as a local dev server would send. `is_routable_ip` drops such an address. The IP is sent base64url without padding. +- **`pu` is the page origin**, not the full URL. +- **`Expires` response header gives the lifetime** of the returned IDs. A 204 means no match. + +## Global Constraints + +- Follow `AGENTS.md`: `error-stack` with `derive_more::Display`, `expect("should ...")`, no local imports, comments above code, doc comment on every public item, Arrange-Act-Assert tests. +- Backtick `LiveIntent`, `OpenRTB` and other CamelCase names in doc comments (`clippy::doc_markdown`). +- Tests use fictional data: `www.example.com`, documentation IPs (`192.0.2.0/24`), fake IDs (`li-example`, `fpi-example`). LiveIntent's public hostnames and the `OpenRTB` source domains in the attribute table may appear. +- Never log the `_lc2_fpi` value, UIDs, consent strings, or the request URL with its query. +- LiveIntent's IAB GVL ID is **148**. All UIDs use `atype` **3**. +- Quick test: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveintent`. Gates: `cargo test-fastly`, `cargo clippy-fastly`. + +## File Structure + +| File | Change | Responsibility | +| ----------------------------------------------------------- | ------ | ------------------------------------------------------- | +| `crates/trusted-server-core/src/integrations/liveintent.rs` | Create | Config, attribute table, provider, registration, tests | +| `crates/trusted-server-core/src/integrations/mod.rs` | Modify | `pub mod liveintent;` and its `IntegrationBuilder` | +| `crates/trusted-server-core/src/config.rs` | Modify | Runtime and deploy validation, deploy-validated ID list | +| `docs/guide/integrations/identity-vendors.md` | Modify | Table row and `## LiveIntent` section | +| `CHANGELOG.md` | Modify | Vendor bullet | + +--- + +### Task 1: Config, attribute table and descriptor + +The attribute table is the one place that knows which `OpenRTB` source each LiveIntent attribute carries. `LiveIntentConfig::attributes()` resolves the configured names against it, refusing an unknown name, and both the descriptor (sources) and the provider (query and response mapping) use the result. + +**Files:** + +- Create: `crates/trusted-server-core/src/integrations/liveintent.rs` +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (add `pub mod liveintent;` in alphabetical order) + +**Interfaces:** + +- Produces: `pub(crate) const LIVEINTENT_INTEGRATION_ID: &str = "liveintent"`, `pub struct LiveIntentConfig { enabled, publisher_id: String, distributor_id: Option, endpoint: String, resolve: Vec }`, `fn LiveIntentConfig::attributes(&self) -> Result, Report>`, `pub struct LiveIntentProvider`, `pub fn LiveIntentProvider::new(&LiveIntentConfig) -> Result>`. + +- [ ] **Step 1: Write the failing tests** + +Create `liveintent.rs` with the tests module only: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::identity::test_support::{ + ConformanceCases, HarnessSetup, ProviderHarness, TestConsent, + check_descriptor_ignores_secrets, + }; + + fn config(resolve: &[&str]) -> LiveIntentConfig { + LiveIntentConfig { + enabled: true, + publisher_id: "12345".to_owned(), + distributor_id: None, + endpoint: default_endpoint(), + resolve: resolve.iter().map(|name| (*name).to_owned()).collect(), + } + } + + fn provider(resolve: &[&str]) -> LiveIntentProvider { + LiveIntentProvider::new(&config(resolve)).expect("should build the provider") + } + + #[test] + fn rejects_an_unknown_attribute() { + assert!( + LiveIntentProvider::new(&config(&["unknown"])).is_err(), + "should refuse an attribute outside the table" + ); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets(&config(&["nonId"]), &config(&["nonId"])); + } +} +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveintent` +Expected: FAIL to compile (`LiveIntentConfig` not found). + +- [ ] **Step 3: Implement the config and descriptor** + +Prepend to `liveintent.rs`. The `use` block is the module's final one; items used only by later tasks warn as unused until then. + +```rust +//! `LiveIntent` identity provider: resolves `LiveIntent`'s IDs server side. +//! +//! Calls `LiveIntent`'s identity exchange with the visitor's first-party +//! `LiveIntent` cookie, consent and network signals, and asks for the +//! attributes the operator configured. Each attribute `LiveIntent` returns +//! maps to one `OpenRTB` source. + +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use error_stack::Report; +use http::StatusCode; +use serde::Deserialize; +use serde_json::{Map, Value, json}; +use validator::Validate; + +use crate::ec::identity::{ + ConsentPolicy, IdentityProvider, IdentityProviderConfig, IdentityProviderError, Input, + ProviderDescriptor, ProviderRequest, ProviderResponse, Resolution, ResolveInput, ResolvedEid, + ResolvedUid, Resolves, SourceDomain, +}; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistration; +use crate::integrations::vendor_request::is_routable_ip; +use crate::settings::{IntegrationConfig, Settings}; + +/// Integration ID of the `LiveIntent` provider. +pub(crate) const LIVEINTENT_INTEGRATION_ID: &str = "liveintent"; + +/// `LiveIntent`'s IAB Global Vendor List ID. +const LIVEINTENT_GVL_ID: u16 = 148; + +/// `LiveIntent`'s first-party ID cookie on the publisher's domain. +const FIRST_PARTY_COOKIE: &str = "_lc2_fpi"; + +/// The attribute that carries `LiveIntent`'s own ID. +const NON_ID_ATTRIBUTE: &str = "nonId"; + +/// Each attribute `LiveIntent` resolves and the source it carries. +const ATTRIBUTE_SOURCES: &[(&str, &str)] = &[ + (NON_ID_ATTRIBUTE, "liveintent.com"), + ("bidswitch", "bidswitch.net"), + ("medianet", "media.net"), + ("magnite", "rubiconproject.com"), + ("index", "liveintent.indexexchange.com"), + ("openx", "openx.net"), + ("pubmatic", "pubmatic.com"), + ("sovrn", "liveintent.sovrn.com"), + ("sharethrough", "sharethrough.com"), + ("sonobi", "liveintent.sonobi.com"), + ("triplelift", "liveintent.triplelift.com"), + ("zetassp", "zeta-ssp.liveintent.com"), + ("vidazoo", "liveintent.vidazoo.com"), + ("nexxen", "liveintent.unrulymedia.com"), + ("uid2", "uidapi.com"), + ("thetradedesk", "adserver.org"), +]; + +/// Config of the `liveintent` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct LiveIntentConfig { + /// Enables the provider. + #[serde(default)] + pub enabled: bool, + /// The publisher ID `LiveIntent` assigned, or `any`. + #[serde(default = "default_publisher_id")] + #[validate(length(min = 1))] + pub publisher_id: String, + /// The distributor ID `LiveIntent` assigned to a distributor of the + /// integration, if any. It takes the place of `prebid` in the path. + pub distributor_id: Option, + /// The identity exchange endpoint. + #[serde(default = "default_endpoint")] + #[validate(url)] + pub endpoint: String, + /// The attributes to resolve, from the table in this module. Each one's + /// source needs an `[[ec.partners]]` entry. Default: `nonId`. + #[serde(default = "default_resolve")] + #[validate(length(min = 1))] + pub resolve: Vec, +} + +fn default_publisher_id() -> String { + "any".to_owned() +} + +fn default_endpoint() -> String { + "https://idx.liadm.com/idex".to_owned() +} + +fn default_resolve() -> Vec { + vec![NON_ID_ATTRIBUTE.to_owned()] +} + +impl IntegrationConfig for LiveIntentConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl LiveIntentConfig { + /// The configured attributes and their sources, in config order. + fn attributes(&self) -> Result, Report> { + self.resolve + .iter() + .map(|attribute| { + let (name, source) = ATTRIBUTE_SOURCES + .iter() + .find(|(name, _)| name == attribute) + .ok_or_else(|| { + IdentityProviderError::config(format!( + "unknown LiveIntent attribute `{attribute}`" + )) + })?; + Ok(((*name).to_owned(), source.parse()?)) + }) + .collect() + } +} + +impl IdentityProviderConfig for LiveIntentConfig { + fn build_descriptor(&self) -> Result> { + let mut builder = ProviderDescriptor::builder(LIVEINTENT_INTEGRATION_ID.parse()?); + for (_, source) in self.attributes()? { + builder = builder.with_source(source); + } + builder + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_wants([Input::ClientIp, Input::PageUrl]) + .with_cookie(FIRST_PARTY_COOKIE), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(LIVEINTENT_GVL_ID)) + .build() + } +} + +/// The `LiveIntent` provider. +pub struct LiveIntentProvider { + descriptor: ProviderDescriptor, + path: String, + distributor_id: Option, + attributes: Vec<(String, SourceDomain)>, +} + +impl LiveIntentProvider { + /// Builds the provider from its validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config names an + /// unknown attribute or cannot describe a valid provider. + pub fn new(config: &LiveIntentConfig) -> Result> { + let distributor = config.distributor_id.as_deref().unwrap_or("prebid"); + Ok(Self { + descriptor: config.build_descriptor()?, + path: format!("{distributor}/{}", config.publisher_id), + distributor_id: config.distributor_id.clone(), + attributes: config.attributes()?, + }) + } +} + +impl IdentityProvider for LiveIntentProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } +} +``` + +The first configured attribute's source is the descriptor's primary source. Because an unknown attribute fails `build_descriptor`, `ts config validate` refuses it at deploy time too. + +- [ ] **Step 4: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveintent` +Expected: PASS (2 tests). + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/liveintent.rs crates/trusted-server-core/src/integrations/mod.rs +git commit -S --signoff -m "Describe the LiveIntent provider from its configured attributes" +``` + +--- + +### Task 2: Build the identity exchange request + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/liveintent.rs` + +**Interfaces:** + +- Consumes: `ResolveInput::{endpoint, publisher_domain, cookie, consent, page_url, client_ip}`, `ProviderRequest::{get, try_with_header, with_query}`, `is_routable_ip`. +- Produces: `LiveIntentProvider::request`. + +- [ ] **Step 1: Write the failing tests** + +Add to the tests module: + +```rust + #[test] + fn calls_the_exchange_with_cookie_consent_and_attributes() { + let provider = provider(&["nonId", "thetradedesk"]); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_page_url("https://www.example.com/article?x=1") + .with_cookie(FIRST_PARTY_COOKIE, "fpi-example") + .with_consent(TestConsent::default().with_us_privacy("1YNN")); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + let url = request.url(); + assert_eq!( + url.path(), + "/idex/prebid/12345", + "should address the publisher" + ); + assert_eq!( + request + .headers() + .get("origin") + .map(http::HeaderValue::as_bytes), + Some(b"https://www.example.com".as_slice()), + "should send the publisher origin, which LiveIntent requires" + ); + let query: Vec<(String, String)> = url.query_pairs().into_owned().collect(); + let value = |name: &str| { + query + .iter() + .filter(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + .collect::>() + }; + assert_eq!( + value("duid"), + ["fpi-example"], + "should send the first-party ID" + ); + assert_eq!(value("us_privacy"), ["1YNN"], "should send US privacy"); + assert_eq!( + value("pu"), + ["https://www.example.com"], + "should send the page origin" + ); + assert_eq!( + value("pip"), + [URL_SAFE_NO_PAD.encode("192.0.2.10")], + "should send the encoded IP" + ); + assert_eq!( + value("resolve"), + ["nonId", "thetradedesk"], + "should ask for each configured attribute" + ); + } + + #[test] + fn leaves_out_a_loopback_client_ip() { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider).with_client_ip("127.0.0.1"); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + assert!( + !request + .url() + .query_pairs() + .any(|(key, _)| key == "pip" || key == "pip6"), + "should not send an address LiveIntent cannot match" + ); + } +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveintent` +Expected: the two new tests FAIL with `NotImplemented("request")`. + +- [ ] **Step 3: Implement `request`** + +Add to `impl IdentityProvider for LiveIntentProvider`: + +```rust + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let mut url = input.endpoint()?.clone(); + url.path_segments_mut() + .map_err(|_| IdentityProviderError::config("endpoint cannot hold a path"))? + .pop_if_empty() + .extend(self.path.split('/')); + let mut request = ProviderRequest::get(&url) + .try_with_header("origin", &format!("https://{}", input.publisher_domain()))?; + if let Some(distributor) = &self.distributor_id { + request = request.with_query("did", distributor); + } + if let Some(fpid) = input.cookie(FIRST_PARTY_COOKIE)? { + request = request.with_query("duid", fpid); + } + let consent = input.consent(); + request = request.with_query("gdpr", if consent.gdpr_applies() { "1" } else { "0" }); + if let Some(tc_string) = consent.raw_tc_string() { + request = request.with_query("gdpr_consent", tc_string); + } + if let Some(us_privacy) = consent.raw_us_privacy() { + request = request.with_query("us_privacy", us_privacy); + } + if let Some(gpp) = consent.raw_gpp_string() { + request = request.with_query("gpp_s", gpp); + } + if let Some(ids) = consent.gpp_section_ids() { + let ids: Vec = ids.iter().map(u16::to_string).collect(); + request = request.with_query("gpp_as", &ids.join(",")); + } + if let Ok(page) = input.page_url() { + request = request.with_query("pu", page.origin().ascii_serialization().as_str()); + } + if let Ok(ip) = input.client_ip() + && is_routable_ip(ip) + { + let name = if ip.is_ipv4() { "pip" } else { "pip6" }; + request = request.with_query(name, &URL_SAFE_NO_PAD.encode(ip.to_string())); + } + for (attribute, _) in &self.attributes { + request = request.with_query("resolve", attribute); + } + Ok(Some(request)) + } +``` + +`input.cookie(..)?` propagates only `UndeclaredInput`, which cannot happen because the descriptor declares `_lc2_fpi`; an absent cookie is `Ok(None)`. + +- [ ] **Step 4: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveintent` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/liveintent.rs +git commit -S --signoff -m "Call the LiveIntent identity exchange with origin, first-party ID and consent" +``` + +--- + +### Task 3: Map the reply to sources + +The reply is a flat object, for example `{"nonId": "...", "segments": [...], "thetradedesk": "..."}`. For each configured attribute whose source this call resolves, a non-empty string value becomes one EID with `atype` 3. `nonId` carries LiveIntent's `segments` as EID-level `ext`. Every other attribute names LiveIntent as `provider` in its UID `ext`; the Trade Desk ID also carries `rtiPartner: "TDID"` and `matcher: "liveintent.com"`, the shape bidders expect for an `adserver.org` ID from LiveIntent. A future `Expires` header sets each EID's expiry. A 204 is no match. + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/liveintent.rs` + +**Interfaces:** + +- Consumes: `ProviderResponse::{status, success_json, header}`, `ResolveInput::{sources, now}`, `ResolvedEid::{with_ext, with_matcher, with_expires_in}`. +- Produces: `LiveIntentProvider::response`. + +- [ ] **Step 1: Write the failing tests** + +Add to the tests module: + +```rust + #[test] + fn maps_attributes_to_sources() { + let provider = provider(&["nonId", "thetradedesk", "pubmatic"]); + let mut harness = ProviderHarness::new(&provider); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ + "nonId": "li-example", + "segments": ["s1"], + "thetradedesk": "tdid-example", + "bidswitch": "not-configured" + }), + ) + .expect("should parse"); + + let eids = resolution.eids(); + assert_eq!( + eids.len(), + 2, + "should map only configured attributes it returned" + ); + assert_eq!( + eids[0].source().as_str(), + "liveintent.com", + "should map nonId" + ); + assert_eq!( + eids[0].ext(), + Some(&json!({ "segments": ["s1"] })), + "should keep segments" + ); + assert_eq!( + eids[1].source().as_str(), + "adserver.org", + "should map the TDID" + ); + assert_eq!( + eids[1].matcher(), + Some("liveintent.com"), + "should name the matcher" + ); + } + + #[test] + fn no_content_is_no_match() { + let provider = provider(&["nonId"]); + let mut harness = ProviderHarness::new(&provider); + harness.request().expect("should build a request"); + + let resolution = harness + .respond_raw(StatusCode::NO_CONTENT, &[], b"") + .expect("should accept 204"); + + assert!(resolution.is_no_match(), "should be no match"); + } + + #[test] + fn conforms() { + ProviderHarness::new(&provider(&["nonId"])).check_conformance( + &ConformanceCases::default().with_resolve_case( + HarnessSetup::default().with_cookie(FIRST_PARTY_COOKIE, "fpi-example"), + StatusCode::OK, + &json!({ "nonId": "li-example" }), + ), + ); + } +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveintent` +Expected: the three new tests FAIL with `NotImplemented("response")`. + +- [ ] **Step 3: Implement `response` and its helpers** + +Add to `impl IdentityProvider for LiveIntentProvider`, after `request`: + +```rust + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + if response.status() == StatusCode::NO_CONTENT { + return Ok(Resolution::no_match()); + } + let reply: Map = response.success_json()?; + let segments = reply.get("segments").cloned(); + let lifetime = expires_in(response, input); + + let eids: Vec = self + .attributes + .iter() + .filter(|(_, source)| input.sources().any(|(wanted, _)| wanted == source)) + .filter_map(|(attribute, source)| { + let id = reply.get(attribute)?.as_str().filter(|id| !id.is_empty())?; + let mut eid = ResolvedEid::new(source, resolved_uid(attribute, id)); + if attribute == NON_ID_ATTRIBUTE + && let Some(segments) = &segments + { + eid = eid.with_ext(json!({ "segments": segments })); + } + if attribute == "thetradedesk" { + eid = eid.with_matcher("liveintent.com"); + } + if let Some(lifetime) = lifetime { + eid = eid.with_expires_in(lifetime); + } + Some(eid) + }) + .collect(); + Ok(if eids.is_empty() { + Resolution::no_match() + } else { + Resolution::found(eids) + }) + } +``` + +The `input.sources()` filter keeps the result to the sources this call is resolving, so a source the operator narrowed away with `[ec.identity.providers.liveintent].sources` is never returned. + +After the `impl` block: + +```rust +fn resolved_uid(attribute: &str, id: &str) -> ResolvedUid { + let uid = ResolvedUid::new(id).with_atype(3); + match attribute { + NON_ID_ATTRIBUTE => uid, + "thetradedesk" => { + uid.with_ext(json!({ "rtiPartner": "TDID", "provider": "liveintent.com" })) + } + _ => uid.with_ext(json!({ "provider": "liveintent.com" })), + } +} + +/// The lifetime `LiveIntent` gives in its `Expires` header, when it is a +/// valid HTTP date in the future. +fn expires_in(response: &ProviderResponse, input: &ResolveInput<'_>) -> Option { + let expires = response.header("expires")?; + let at = chrono::DateTime::parse_from_rfc2822(expires) + .ok()? + .timestamp(); + let seconds = u64::try_from(at).ok()?.checked_sub(input.now().as_secs())?; + (seconds > 0).then(|| Duration::from_secs(seconds)) +} +``` + +`input.now()` (not the system clock) keeps the provider pure and deterministic in the harness. + +- [ ] **Step 4: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveintent` +Expected: PASS (7 tests). + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/liveintent.rs +git commit -S --signoff -m "Map LiveIntent attributes to their OpenRTB sources" +``` + +--- + +### Task 4: Register, validate and document + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/liveintent.rs` (`register`) +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (`builders()`) +- Modify: `crates/trusted-server-core/src/config.rs` +- Modify: `docs/guide/integrations/identity-vendors.md`, `CHANGELOG.md` + +**Interfaces:** + +- Produces: `pub fn register(settings: &Settings) -> Result, Report>`. + +- [ ] **Step 1: Write `register` and add the builder** + +Append to `liveintent.rs`, before the tests module: + +```rust +/// Registers the `LiveIntent` provider when `[integrations.liveintent]` is +/// enabled. +/// +/// # Errors +/// +/// Returns an error when the config does not parse or describes an invalid +/// provider. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = + settings.integration_config::(LIVEINTENT_INTEGRATION_ID)? + else { + return Ok(None); + }; + log::info!( + "Registering LiveIntent identity provider (attributes={})", + config.resolve.join(",") + ); + Ok(Some( + IntegrationRegistration::builder(LIVEINTENT_INTEGRATION_ID) + .try_with_identity_provider(LiveIntentProvider::new(&config))? + .without_js() + .build(), + )) +} +``` + +In `integrations/mod.rs` `builders()`, add: + +```rust + IntegrationBuilder { + id: liveintent::LIVEINTENT_INTEGRATION_ID, + build: liveintent::register, + }, +``` + +- [ ] **Step 2: Run the deploy-coverage test and see it fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config::tests::deploy_validation_covers_registered_integration_builders` +Expected: FAIL with `missing_ids` containing `"liveintent"`. + +- [ ] **Step 3: Validate at runtime and at deploy** + +In `config.rs`, import `liveintent::{LIVEINTENT_INTEGRATION_ID, LiveIntentConfig}`. Add `LIVEINTENT_INTEGRATION_ID,` to `DEPLOY_VALIDATED_INTEGRATION_IDS`. In `identity_descriptors_for_deploy`: + +```rust + descriptors.extend(validate_identity_integration::( + settings, + LIVEINTENT_INTEGRATION_ID, + )?); +``` + +If `validate_identity_integration` still carries the framework's temporary `#[allow(dead_code)]`, remove it and its comment. In `validate_enabled_integrations`: + +```rust + validate_integration::(settings, LIVEINTENT_INTEGRATION_ID)?; +``` + +- [ ] **Step 4: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config` +Expected: PASS. + +- [ ] **Step 5: Document** + +In `docs/guide/integrations/identity-vendors.md`, add this table row: + +```markdown +| `liveintent` | LiveIntent | Resolve | One per configured attribute (see below) | Nothing (`_lc2_fpi` improves match) | +``` + +and this section: + +````markdown +## LiveIntent + +```toml +[integrations.liveintent] +enabled = true +publisher_id = "12345" +# distributor_id = "did-0000" +resolve = ["nonId", "thetradedesk"] +``` + +`resolve` lists the attributes to ask for. Each attribute maps to one source: + +| Attribute | Source | +| -------------- | ------------------------------ | +| `nonId` | `liveintent.com` | +| `bidswitch` | `bidswitch.net` | +| `medianet` | `media.net` | +| `magnite` | `rubiconproject.com` | +| `index` | `liveintent.indexexchange.com` | +| `openx` | `openx.net` | +| `pubmatic` | `pubmatic.com` | +| `sovrn` | `liveintent.sovrn.com` | +| `sharethrough` | `sharethrough.com` | +| `sonobi` | `liveintent.sonobi.com` | +| `triplelift` | `liveintent.triplelift.com` | +| `zetassp` | `zeta-ssp.liveintent.com` | +| `vidazoo` | `liveintent.vidazoo.com` | +| `nexxen` | `liveintent.unrulymedia.com` | +| `uid2` | `uidapi.com` | +| `thetradedesk` | `adserver.org` | + +Only the configured attributes are declared, so only their sources need partner entries. LiveIntent needs vendor consent for GVL ID 148. When LiveIntent's first-party `_lc2_fpi` cookie is on the request, it is sent as `duid`. +```` + +In `CHANGELOG.md` under `### Added`, insert "LiveIntent (`[integrations.liveintent]`)" into the vendor bullet's list of providers, or, if there is no vendor bullet yet, add: + +```markdown +- Server-side identity provider for LiveIntent (`[integrations.liveintent]`). Each source it writes needs an `[[ec.partners]]` entry. See [Identity Vendors](docs/guide/integrations/identity-vendors.md). +``` + +- [ ] **Step 6: Run the gates** + +Run: `cargo fmt --all -- --check && cargo test-fastly && cargo clippy-fastly` +Run: `cd docs && npm run format` +Run: `docs/node_modules/.bin/prettier --config docs/.prettierrc --check CHANGELOG.md` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/ docs/guide/integrations/identity-vendors.md CHANGELOG.md +git commit -S --signoff -m "Register the LiveIntent identity provider and validate its config" +``` + +## Self-Review Notes + +- Path shape, required `Origin`, `did` only when configured, `duid` from `_lc2_fpi`, routable `pip`: Task 2. +- `Expires` lifetime and 204 no match: Task 3. +- Unknown attribute refused at build and deploy: Task 1. diff --git a/docs/superpowers/plans/2026-10-28-identity-provider-liveramp.md b/docs/superpowers/plans/2026-10-28-identity-provider-liveramp.md new file mode 100644 index 0000000000..e4869acc21 --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-identity-provider-liveramp.md @@ -0,0 +1,636 @@ +# LiveRamp ATS Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve and refresh LiveRamp ATS envelopes at the edge with a server-side `IdentityProvider` under `[integrations.liveramp]`, writing them to the `liveramp.com` source. + +**Architecture:** `integrations/liveramp.rs` holds `LiveRampConfig`, `LiveRampProvider` and `register`. An initial lookup sends the visitor's hashed email from `ts-hem`; a refresh sends the stored envelope. The provider marks its call as initial or refresh in the request's call context, because LiveRamp's 204 means "no match" for one and "envelope no longer accepted" for the other. The descriptor declares expiring IDs, and the response sets the envelope's expiry from the visitor's region: 15 days in California, 30 days elsewhere. + +**Tech Stack:** Rust 1.95 (edition 2024), `trusted-server-core`, `serde`, `http`, `validator`, `error-stack` (all already dependencies). + +**Spec:** `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`, section 12.1 (LiveRamp ATS) and sections 4.2 to 4.6 (the provider API). Section 12.1 marks the response shape and consent parameter names as assumed; this plan keeps them and the operator doc asks operators to confirm them with LiveRamp. Operator page: `docs/guide/integrations/identity-vendors.md`. + +**Prerequisite:** `docs/superpowers/plans/2026-10-28-identity-provider-inputs.md`. This plan needs `Input::HashedEmail` from the `ts-hem` cookie for the initial lookup, and adds its section to the operator page that plan creates. + +## Design notes + +- **Initial versus refresh.** `input.is_refresh(source)` with a stored envelope picks a refresh (`it=19`, `iv=`); anything else is an initial lookup (`it=4`, `iv=`). The framework attempts a refresh for an expired envelope too, and LiveRamp answers 204 for one it no longer accepts, so a refresh 204 invalidates the stored envelope while an initial 204 is just no match. +- **Hashed email is needed only for `Initial`** (`with_needs_for(ResolveKind::Initial, [Input::HashedEmail])`). A visitor without `ts-hem` is never sent an initial lookup, but a stored envelope still refreshes on later visits. +- **Client IP and region are needed.** The IP is forwarded in `X-Forwarded-For`; the region decides the envelope's lifetime. +- **Lifetime.** Envelopes expire after 15 days for visitors in California (`US`/`CA`) and 30 days elsewhere. The provider sets no refresh hint, so the operator's `refresh_after_sec` paces refreshes. +- **Product selection.** The reply lists envelope products; the provider keeps the regular ATS envelope (`type` 19, `source` `envelopeLiveramp`) with a non-empty value. + +## Global Constraints + +- Follow `AGENTS.md`: `error-stack` with `derive_more::Display`, `expect("should ...")`, no local imports, comments above code, doc comment on every public item, Arrange-Act-Assert tests. +- Backtick `LiveRamp` and other CamelCase names in doc comments (`clippy::doc_markdown`). +- Tests use fictional data: `ats.example.com` endpoint, documentation IPs, fake envelopes (`envelope-example`). LiveRamp's public default endpoint host and the `liveramp.com` source may appear. +- Never log envelopes, the hashed email, the client IP or consent strings. The placement ID is an operator identifier, not a secret. +- LiveRamp's IAB GVL ID is **97**. UIDs use `atype` **3**. Expiry: **15** days in `US`/`CA`, **30** days elsewhere. +- Quick test: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveramp`. Gates: `cargo test-fastly`, `cargo clippy-fastly`. + +## File Structure + +| File | Change | Responsibility | +| --------------------------------------------------------- | ------ | ------------------------------------------------------- | +| `crates/trusted-server-core/src/integrations/liveramp.rs` | Create | Config, descriptor, provider, registration, tests | +| `crates/trusted-server-core/src/integrations/mod.rs` | Modify | `pub mod liveramp;` and its `IntegrationBuilder` | +| `crates/trusted-server-core/src/config.rs` | Modify | Runtime and deploy validation, deploy-validated ID list | +| `docs/guide/integrations/identity-vendors.md` | Modify | Table row and `## LiveRamp ATS` section | +| `CHANGELOG.md` | Modify | Vendor bullet | + +--- + +### Task 1: Config, descriptor and request + +**Files:** + +- Create: `crates/trusted-server-core/src/integrations/liveramp.rs` +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (add `pub mod liveramp;` in alphabetical order) + +**Interfaces:** + +- Consumes: `ResolveInput::{primary_source, stored, is_refresh, hashed_email, consent, endpoint, publisher_domain, client_ip}`, `ProviderRequest::{get, with_query, try_with_call_context, try_with_header}`, `HashedEmail::to_hex`. +- Produces: `pub(crate) const LIVERAMP_INTEGRATION_ID: &str = "liveramp"`, `pub struct LiveRampConfig { enabled, endpoint: String, placement_id: String }`, `pub struct LiveRampProvider`, `pub fn LiveRampProvider::new(&LiveRampConfig) -> Result>`, `enum LiveRampCall { Initial, Refresh }` (call context). + +- [ ] **Step 1: Write the failing tests** + +Create `liveramp.rs` with the tests module only: + +```rust +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::ec::identity::test_support::{ + ConformanceCases, HarnessSetup, ProviderHarness, check_descriptor_ignores_secrets, + }; + use crate::ec::identity::{HashedEmail, SourceDomain, StoredId}; + + fn config() -> LiveRampConfig { + LiveRampConfig { + enabled: true, + endpoint: "https://ats.example.com/api/identity/v2/envelope".to_owned(), + placement_id: "placement-1".to_owned(), + } + } + + fn provider() -> LiveRampProvider { + LiveRampProvider::new(&config()).expect("should build the provider") + } + + fn source() -> SourceDomain { + "liveramp.com".parse().expect("should parse the source") + } + + fn query(request: &ProviderRequest, name: &str) -> Option { + request + .url() + .query_pairs() + .find(|(key, _)| key == name) + .map(|(_, value)| value.into_owned()) + } + + #[test] + fn initial_lookup_sends_the_hashed_email() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_region("US", Some("NY")) + .with_hashed_email(HashedEmail::for_test([0xab; 32])); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + + assert_eq!( + query(&request, "it").as_deref(), + Some("4"), + "should be a HEM lookup" + ); + assert_eq!( + query(&request, "iv"), + Some("ab".repeat(32)), + "should send the HEM hex" + ); + assert_eq!( + request + .headers() + .get("x-forwarded-for") + .map(http::HeaderValue::as_bytes), + Some(b"192.0.2.10".as_slice()), + "should forward the client IP" + ); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets(&config(), &config()); + } +} +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveramp` +Expected: FAIL to compile (`LiveRampConfig` not found). + +- [ ] **Step 3: Implement the config and descriptor** + +Prepend to `liveramp.rs`. The `use` block is the module's final one; items used only by Task 2 warn as unused until then. + +```rust +//! `LiveRamp` ATS identity provider: resolves and refreshes ATS envelopes +//! server side. +//! +//! An initial lookup sends the hashed email from the `ts-hem` cookie. A +//! refresh sends the stored envelope. Envelopes expire after 15 days in +//! California and 30 days elsewhere. The response shape and the consent +//! parameter names are assumed (spec 12.1) and should be confirmed with +//! `LiveRamp`. + +use std::time::Duration; + +use error_stack::Report; +use http::StatusCode; +use serde::{Deserialize, Serialize}; +use validator::Validate; + +use crate::ec::identity::{ + ConsentPolicy, IdentityProvider, IdentityProviderConfig, IdentityProviderError, Input, + ProviderDescriptor, ProviderRequest, ProviderResponse, Resolution, ResolveInput, ResolveKind, + ResolvedEid, ResolvedUid, Resolves, +}; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistration; +use crate::integrations::vendor_request::is_routable_ip; +use crate::settings::{IntegrationConfig, Settings}; + +/// Integration ID of the `LiveRamp` provider. +pub(crate) const LIVERAMP_INTEGRATION_ID: &str = "liveramp"; + +/// `LiveRamp`'s IAB Global Vendor List ID. +const LIVERAMP_GVL_ID: u16 = 97; + +/// The regular ATS envelope product. +const ATS_ENVELOPE_TYPE: u32 = 19; +const ATS_ENVELOPE_SOURCE: &str = "envelopeLiveramp"; + +/// Identifier types: 4 is a hashed email, 19 an envelope refresh. +const IT_HASHED_EMAIL: &str = "4"; +const IT_ENVELOPE: &str = "19"; + +/// Consent type 4: an IAB TCF v2 string. +const CT_TCF: &str = "4"; + +const SECONDS_PER_DAY: u64 = 86_400; + +/// Config of the `liveramp` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct LiveRampConfig { + /// Enables the provider. + #[serde(default)] + pub enabled: bool, + /// The ATS envelope endpoint. + #[serde(default = "default_endpoint")] + #[validate(url)] + pub endpoint: String, + /// `LiveRamp` placement ID. An operator identifier, not a secret. + #[validate(length(min = 1))] + pub placement_id: String, +} + +fn default_endpoint() -> String { + "https://api.rlcdn.com/api/identity/v2/envelope".to_owned() +} + +impl IntegrationConfig for LiveRampConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl IdentityProviderConfig for LiveRampConfig { + fn build_descriptor(&self) -> Result> { + ProviderDescriptor::builder(LIVERAMP_INTEGRATION_ID.parse()?) + .with_source("liveramp.com".parse()?) + .with_expiring_ids() + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_needs([Input::ClientIp, Input::Region]) + .with_needs_for(ResolveKind::Initial, [Input::HashedEmail]), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(LIVERAMP_GVL_ID)) + .build() + } +} +``` + +- [ ] **Step 4: Implement the provider and its request** + +Append, before the tests module: + +```rust +/// The `LiveRamp` ATS provider. +pub struct LiveRampProvider { + descriptor: ProviderDescriptor, + placement_id: String, +} + +impl LiveRampProvider { + /// Builds the provider from its validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config cannot + /// describe a valid provider. + pub fn new(config: &LiveRampConfig) -> Result> { + Ok(Self { + descriptor: config.build_descriptor()?, + placement_id: config.placement_id.clone(), + }) + } +} + +/// Whether the call was an initial lookup or a refresh, carried from +/// `request` to `response`: a 204 means different things for each. +#[derive(Debug, Serialize, Deserialize)] +enum LiveRampCall { + Initial, + Refresh, +} + +impl IdentityProvider for LiveRampProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let source = input.primary_source(); + let (call, identifier_type, identifier_value) = match input.stored(source) { + Some(stored) if input.is_refresh(source) => { + (LiveRampCall::Refresh, IT_ENVELOPE, stored.uid().to_owned()) + } + _ => ( + LiveRampCall::Initial, + IT_HASHED_EMAIL, + input.hashed_email()?.to_hex(), + ), + }; + let consent = input.consent(); + let mut request = ProviderRequest::get(input.endpoint()?) + .with_query("pid", &self.placement_id) + .with_query("it", identifier_type) + .with_query("iv", &identifier_value) + .with_query("atype", "3") + .try_with_call_context(&call)? + .try_with_header("origin", &format!("https://{}", input.publisher_domain()))?; + // A loopback or private address identifies nobody to LiveRamp. + let client_ip = input.client_ip()?; + if is_routable_ip(client_ip) { + request = request.try_with_header("x-forwarded-for", &client_ip.to_string())?; + } + if let Some(tc_string) = consent.raw_tc_string() { + request = request.with_query("ct", CT_TCF).with_query("cv", tc_string); + } + if let Some(gpp) = consent.raw_gpp_string() { + request = request.with_query("gpp", gpp); + } + if let Some(ids) = consent.gpp_section_ids() { + let ids: Vec = ids.iter().map(u16::to_string).collect(); + request = request.with_query("gpp_sid", &ids.join(",")); + } + Ok(Some(request)) + } +} +``` + +`input.hashed_email()?` in the initial arm is safe: eligibility only schedules an `Initial` when the hashed email is available, because the descriptor needs it for that kind. + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveramp` +Expected: PASS (2 tests). + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/liveramp.rs crates/trusted-server-core/src/integrations/mod.rs +git commit -S --signoff -m "Build LiveRamp ATS lookup and refresh requests" +``` + +--- + +### Task 2: Parse envelopes, set the lifetime, invalidate a refused refresh + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/liveramp.rs` + +**Interfaces:** + +- Consumes: `ProviderResponse::{call_context, status, success_json}`, `ResolveInput::region` (`Region::is(country, subdivision)`), `ResolvedEid::with_expires_in`, `Resolution::{found, no_match, invalidate}`. +- Produces: `LiveRampProvider::response`. + +- [ ] **Step 1: Write the failing tests** + +Add to the tests module: + +```rust + #[test] + fn california_envelopes_expire_after_15_days() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_region("US", Some("CA")) + .with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let resolution = harness + .respond( + StatusCode::OK, + &json!({ "envelopes": [ + { "type": 19, "source": "envelopeLiveramp", "value": "envelope-example" } + ]}), + ) + .expect("should parse"); + + let eid = &resolution.eids()[0]; + assert_eq!( + eid.uids()[0].id(), + "envelope-example", + "should keep the envelope" + ); + assert_eq!( + eid.expiry().map(|when| harness.resolve(when)), + Some(crate::ec::identity::UnixTime::from_secs( + 1_760_000_000 + 15 * SECONDS_PER_DAY + )), + "should expire after 15 days in California" + ); + } + + #[test] + fn a_refused_refresh_invalidates_the_envelope() { + let provider = provider(); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_region("US", Some("NY")) + .with_stored( + StoredId::for_test(&source(), "old-envelope") + .expiring_at(crate::ec::identity::UnixTime::from_secs(1)), + ); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + assert_eq!( + query(&request, "it").as_deref(), + Some("19"), + "should be a refresh" + ); + + let resolution = harness + .respond_raw(StatusCode::NO_CONTENT, &[], b"") + .expect("should accept 204"); + assert_eq!( + resolution.invalidated(), + [source()], + "should invalidate the envelope" + ); + } + + #[test] + fn conforms() { + ProviderHarness::new(&provider()).check_conformance( + &ConformanceCases::default().with_resolve_case( + HarnessSetup::default() + .with_client_ip("192.0.2.10") + .with_region("US", Some("NY")) + .with_hashed_email(HashedEmail::for_test([0xab; 32])), + StatusCode::OK, + &json!({ "envelopes": [ + { "type": 19, "source": "envelopeLiveramp", "value": "envelope-example" } + ]}), + ), + ); + } +``` + +`1_760_000_000` is the harness's fixed "now". The refresh test stores an envelope that expired at second 1: the framework still schedules a `Refresh` for an expired record of an expiring-ID provider, and the refresh request works without a hashed email. + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveramp` +Expected: the three new tests FAIL with `NotImplemented("response")`. + +- [ ] **Step 3: Add the reply types** + +Append after `LiveRampCall`: + +```rust +#[derive(Deserialize)] +struct EnvelopeReply { + #[serde(default)] + envelopes: Vec, +} + +#[derive(Deserialize)] +struct EnvelopeProduct { + #[serde(rename = "type")] + product_type: u32, + source: String, + value: String, +} +``` + +- [ ] **Step 4: Implement `response`** + +Add to `impl IdentityProvider for LiveRampProvider`, after `request`: + +```rust + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let source = input.primary_source(); + let call: LiveRampCall = response + .call_context()? + .ok_or_else(|| IdentityProviderError::config("missing call context"))?; + if response.status() == StatusCode::NO_CONTENT { + return Ok(match call { + LiveRampCall::Refresh => Resolution::invalidate(source), + LiveRampCall::Initial => Resolution::no_match(), + }); + } + let reply: EnvelopeReply = response.success_json()?; + let Some(envelope) = reply.envelopes.into_iter().find(|product| { + product.product_type == ATS_ENVELOPE_TYPE + && product.source == ATS_ENVELOPE_SOURCE + && !product.value.is_empty() + }) else { + return Ok(Resolution::no_match()); + }; + let lifetime_days = if input.region()?.is("US", "CA") { + 15 + } else { + 30 + }; + let eid = ResolvedEid::new(source, ResolvedUid::new(envelope.value).with_atype(3)) + .with_expires_in(Duration::from_secs(lifetime_days * SECONDS_PER_DAY)); + Ok(Resolution::found([eid])) + } +``` + +A missing call context is a provider bug (the request always sets one), so it is a `Config` error. A non-2xx other than 204 goes through `success_json` as `Rejected` and backs off this visitor only. + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::liveramp` +Expected: PASS (5 tests). + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/liveramp.rs +git commit -S --signoff -m "Store LiveRamp envelopes with region lifetimes and invalidate refused refreshes" +``` + +--- + +### Task 3: Register, validate and document + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/liveramp.rs` (`register`) +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (`builders()`) +- Modify: `crates/trusted-server-core/src/config.rs` +- Modify: `docs/guide/integrations/identity-vendors.md`, `CHANGELOG.md` + +**Interfaces:** + +- Produces: `pub fn register(settings: &Settings) -> Result, Report>`. + +- [ ] **Step 1: Write `register` and add the builder** + +Append to `liveramp.rs`, before the tests module: + +```rust +/// Registers the `LiveRamp` provider when `[integrations.liveramp]` is +/// enabled. +/// +/// # Errors +/// +/// Returns an error when the config does not parse or describes an invalid +/// provider. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.integration_config::(LIVERAMP_INTEGRATION_ID)? + else { + return Ok(None); + }; + log::info!( + "Registering LiveRamp ATS identity provider (placement={})", + config.placement_id + ); + Ok(Some( + IntegrationRegistration::builder(LIVERAMP_INTEGRATION_ID) + .try_with_identity_provider(LiveRampProvider::new(&config))? + .without_js() + .build(), + )) +} +``` + +In `integrations/mod.rs` `builders()`, add: + +```rust + IntegrationBuilder { + id: liveramp::LIVERAMP_INTEGRATION_ID, + build: liveramp::register, + }, +``` + +- [ ] **Step 2: Run the deploy-coverage test and see it fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config::tests::deploy_validation_covers_registered_integration_builders` +Expected: FAIL with `missing_ids` containing `"liveramp"`. + +- [ ] **Step 3: Validate at runtime and at deploy** + +In `config.rs`, import `liveramp::{LIVERAMP_INTEGRATION_ID, LiveRampConfig}`. Add `LIVERAMP_INTEGRATION_ID,` to `DEPLOY_VALIDATED_INTEGRATION_IDS`. In `identity_descriptors_for_deploy`: + +```rust + descriptors.extend(validate_identity_integration::( + settings, + LIVERAMP_INTEGRATION_ID, + )?); +``` + +If `validate_identity_integration` still carries the framework's temporary `#[allow(dead_code)]`, remove it and its comment. In `validate_enabled_integrations`: + +```rust + validate_integration::(settings, LIVERAMP_INTEGRATION_ID)?; +``` + +- [ ] **Step 4: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config` +Expected: PASS. + +- [ ] **Step 5: Document** + +In `docs/guide/integrations/identity-vendors.md`, add this table row: + +```markdown +| `liveramp` | LiveRamp ATS | Resolve | `liveramp.com` | Hashed email for a first envelope | +``` + +and this section: + +````markdown +## LiveRamp ATS + +```toml +[integrations.liveramp] +enabled = true +placement_id = "placement-1" +# endpoint = "https://api.rlcdn.com/api/identity/v2/envelope" +``` + +A first envelope needs the hashed email. A refresh sends the stored envelope instead, so it works on later visits without one. Envelopes expire after 15 days in California and 30 days elsewhere. Calls need vendor consent for GVL ID 97. The response shape and consent parameter names follow the design spec's assumptions; confirm them with LiveRamp. +```` + +In `CHANGELOG.md` under `### Added`, insert "LiveRamp ATS (`[integrations.liveramp]`)" into the vendor bullet's list of providers, or, if there is no vendor bullet yet, add: + +```markdown +- Server-side identity provider for LiveRamp ATS (`[integrations.liveramp]`). Each source it writes needs an `[[ec.partners]]` entry. See [Identity Vendors](docs/guide/integrations/identity-vendors.md). +``` + +- [ ] **Step 6: Run the gates** + +Run: `cargo fmt --all -- --check && cargo test-fastly && cargo clippy-fastly` +Run: `cd docs && npm run format` +Run: `docs/node_modules/.bin/prettier --config docs/.prettierrc --check CHANGELOG.md` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/ docs/guide/integrations/identity-vendors.md CHANGELOG.md +git commit -S --signoff -m "Register the LiveRamp ATS identity provider and validate its config" +``` + +## Self-Review Notes + +- Initial (`it=4`, HEM) versus refresh (`it=19`, envelope) and the call context: Task 1. +- 204 meaning per call kind, product selection, 15-day California / 30-day lifetime: Task 2. diff --git a/docs/superpowers/plans/2026-10-28-identity-provider-lockr-capture.md b/docs/superpowers/plans/2026-10-28-identity-provider-lockr-capture.md new file mode 100644 index 0000000000..95cad0529f --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-identity-provider-lockr-capture.md @@ -0,0 +1,565 @@ +# Lockr Capture Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let the existing Lockr integration register, behind `[integrations.lockr].capture_identity`, as a capture-only identity provider that removes Lockr-issued IDs from KV when Lockr's API reports a consent revocation. + +**Architecture:** Trusted Server already proxies Lockr's API under `/integrations/lockr/api/...`. A new private module, `integrations/lockr_identity.rs`, implements `IdentityProviderConfig` for the existing `LockrConfig` and an `IdentityProvider` whose descriptor declares capture rules for five exact Lockr API paths and the sources Lockr's tokens carry. The framework offers each proxied 2xx response on those paths to `capture`. Capture is revoke-only: a `revoke-consent` call returns an opt-out, which the framework writes at once. Token responses are parsed into EIDs, but the framework never stores a positive capture; a positive result only clears an earlier Lockr opt-out once Lockr issues again. Lockr's IDs themselves keep reaching KV through browser ingest. + +**Tech Stack:** Rust 1.95 (edition 2024), `trusted-server-core`, `serde`, `http`, `error-stack` (all already dependencies). + +**Spec:** `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`, section 12.2 (Lockr, passive capture), section 4.7 (capture is revoke-only; removals bypass the consent gate; only a real browser's capture writes), section 8 (opt-out clearing) and section 12.5 (`Captures::with_path`, `with_provenance`, `SourceDomain::matches`). Operator page: `docs/guide/integrations/identity-vendors.md`; existing Lockr page: `docs/guide/integrations/lockr.md`. + +**Prerequisite:** `docs/superpowers/plans/2026-10-28-identity-provider-inputs.md`. This plan uses `VendorEid` (`integrations/vendor_eid.rs`) to read the complete EIDs in Lockr's token responses, and adds its section to the operator page that plan creates. + +## Design notes + +- **Revoke-only.** The coordinator's `run_capture` writes a real browser's removals, opt-outs and clears with the removal write, whatever the consent, and counts and discards any positive ID. So the only effect that reaches KV from this provider is the opt-out on `revoke-consent` and, later, its clear. +- **Why parse tokens at all.** For a capture-only provider without a GVL ID, a positive capture is the one signal that the visitor is back: Lockr only issues tokens again after re-consent. The coordinator clears the opt-out when a positive capture passes the consent steps. Parsing also lets `check_conformance` verify the provider's EIDs. +- **Exact paths.** `Captures::with_path(Method::POST, path)` matches each Lockr call exactly, rather than a prefix that would also capture unrelated Lockr calls. +- **Known key mappings only.** Lockr's `ids` map is keyed by token name. Only confirmed names map to sources: `id5id` to `id5-sync.com`, `_lr_env` to `liveramp.com`, `__uid2_advertising_token` to `uidapi.com`. An entry whose EID `source` does not match the mapped source (normalized with `SourceDomain::matches`) is ignored. An expired token (`aimTokens[].identity_expires`, milliseconds) is skipped. +- **Provenance.** ID5 IDs passed through Lockr carry `inserter: "Lockr-For-Publishers"` and `mm: 2`, declared once with `with_provenance` and applied on output. +- **Sources must be listed.** A capture-only provider must list `[ec.identity.providers.lockr].sources`; framework validation enforces it. + +## Global Constraints + +- Follow `AGENTS.md`: `error-stack` with `derive_more::Display`, `expect("should ...")`, no local imports, comments above code, doc comment on every public item, Arrange-Act-Assert tests. +- Tests use fictional data: `app-1` app ID, fake IDs (`ID5*example`), `inserter.example.com`. Public source domains (`id5-sync.com`, `liveramp.com`, `uidapi.com`) may appear. +- Never log tokens, UIDs or the proxied request and response bodies. +- `capture_identity` defaults to `false`; with it off, nothing about the Lockr integration changes. +- Quick test: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::lockr` (this filter covers both `lockr` and `lockr_identity`). Gates: `cargo test-fastly`, `cargo clippy-fastly`. + +## File Structure + +| File | Change | Responsibility | +| --------------------------------------------------------------- | ------ | --------------------------------------------------------------- | +| `crates/trusted-server-core/src/integrations/lockr.rs` | Modify | `capture_identity` flag, `pub(crate)` ID, register the provider | +| `crates/trusted-server-core/src/integrations/lockr_identity.rs` | Create | Descriptor for `LockrConfig`, `LockrIdentityProvider`, tests | +| `crates/trusted-server-core/src/integrations/mod.rs` | Modify | `mod lockr_identity;` | +| `crates/trusted-server-core/src/config.rs` | Modify | Deploy-time descriptor when `capture_identity` is on | +| `docs/guide/integrations/identity-vendors.md` | Modify | Table row and `## Lockr capture` section | +| `CHANGELOG.md` | Modify | Vendor bullet | + +--- + +### Task 1: The flag, the descriptor and revocation capture + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/lockr.rs` (`LOCKR_INTEGRATION_ID` visibility, `LockrConfig`, the test config literal) +- Create: `crates/trusted-server-core/src/integrations/lockr_identity.rs` +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (add `mod lockr_identity;` after `pub mod lockr;`) + +**Interfaces:** + +- Consumes: `Captures::{new, with_path}`, `ProviderDescriptorBuilder::{with_captures, with_source, with_provenance}`, `EidProvenance::{new, with_inserter, with_mm}`, `CaptureInput::path`, `Resolution::opted_out`. +- Produces: `pub(crate) const LOCKR_INTEGRATION_ID: &str = "lockr"`, `pub capture_identity: bool` on `LockrConfig`, `impl IdentityProviderConfig for LockrConfig`, `pub struct LockrIdentityProvider`, `pub fn LockrIdentityProvider::new(&LockrConfig) -> Result>`. + +- [ ] **Step 1: Add the flag** + +In `lockr.rs`, make the ID visible to the crate: + +```rust +pub(crate) const LOCKR_INTEGRATION_ID: &str = "lockr"; +``` + +Add the last field to `LockrConfig`: + +```rust + /// Registers Lockr as a capture-only identity provider that revokes IDs + /// when Lockr's API reports a consent revocation. Needs + /// `[ec.identity.providers.lockr].sources`. Default: off. + #[serde(default)] + pub capture_identity: bool, +``` + +and `capture_identity: false,` to the `LockrConfig { .. }` literal in `lockr.rs`'s tests. + +- [ ] **Step 2: Write the failing tests** + +Create `lockr_identity.rs` with the tests module only, and declare `mod lockr_identity;` in `integrations/mod.rs`: + +```rust +#[cfg(test)] +mod tests { + use http::StatusCode; + use serde_json::json; + + use super::*; + use crate::ec::identity::test_support::{CaptureFixture, ConformanceCases, ProviderHarness}; + + fn config() -> LockrConfig { + serde_json::from_value(json!({ "app_id": "app-1", "capture_identity": true })) + .expect("should parse the Lockr config") + } + + fn provider() -> LockrIdentityProvider { + LockrIdentityProvider::new(&config()).expect("should build the provider") + } + + fn path(endpoint: &str) -> String { + format!("{LOCKR_API}/{endpoint}") + } + + #[test] + fn a_revocation_opts_out() { + let provider = provider(); + + let resolution = ProviderHarness::new(&provider) + .capture(&CaptureFixture::new(Method::POST, &path("revoke-consent"))) + .expect("should capture"); + + assert!(resolution.is_opted_out(), "should opt the visitor out"); + } + + #[test] + fn conforms() { + ProviderHarness::new(&provider()).check_conformance( + &ConformanceCases::default() + .with_capture_case(CaptureFixture::new(Method::POST, &path("revoke-consent"))), + ); + } +} +``` + +- [ ] **Step 3: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::lockr` +Expected: FAIL to compile (`LockrIdentityProvider`, `LOCKR_API` not found). + +- [ ] **Step 4: Implement the descriptor and revocation capture** + +Prepend to `lockr_identity.rs`. The `use` block is the module's final one; items used only by Task 2 (`BTreeMap`, `Deserialize`, `ResolvedEid`, `UnixTime`, `VendorEid`) warn as unused until then. + +```rust +//! Lockr as a capture-only identity provider (spec 12.2). +//! +//! Reads the Lockr API responses Trusted Server already proxies. Capture is +//! revoke-only: a `revoke-consent` call opts the visitor out of Lockr's +//! sources, and a later token issue clears that opt-out. The tokens it +//! reads are not stored; browser ingest carries Lockr's IDs to KV. + +use std::collections::BTreeMap; + +use error_stack::Report; +use http::Method; +use serde::Deserialize; + +use crate::ec::identity::{ + CaptureInput, Captures, EidProvenance, IdentityProvider, IdentityProviderConfig, + IdentityProviderError, ProviderDescriptor, Resolution, ResolvedEid, UnixTime, +}; +use crate::integrations::lockr::{LOCKR_INTEGRATION_ID, LockrConfig}; +use crate::integrations::vendor_eid::VendorEid; + +/// The proxied Lockr API prefix. +const LOCKR_API: &str = "/integrations/lockr/api/publisher/app/v2/identityLockr"; + +/// The Lockr API calls captured, by their last path segment. +const CAPTURED_ENDPOINTS: &[&str] = &[ + "page-view", + "generate-tokens", + "refresh-tokens", + "sync-no-hem-ids", + "revoke-consent", +]; + +/// The endpoint that reports a consent revocation. +const REVOKE_ENDPOINT: &str = "revoke-consent"; + +/// Lockr's token key names and the sources they carry. Only confirmed +/// mappings are listed. +const KEY_SOURCES: &[(&str, &str)] = &[ + ("id5id", "id5-sync.com"), + ("_lr_env", "liveramp.com"), + ("__uid2_advertising_token", "uidapi.com"), +]; + +impl IdentityProviderConfig for LockrConfig { + fn build_descriptor(&self) -> Result> { + let mut captures = Captures::new(); + for endpoint in CAPTURED_ENDPOINTS { + captures = captures.with_path(Method::POST, &format!("{LOCKR_API}/{endpoint}")); + } + let mut builder = + ProviderDescriptor::builder(LOCKR_INTEGRATION_ID.parse()?).with_captures(captures); + for (_, domain) in KEY_SOURCES { + builder = builder.with_source(domain.parse()?); + } + builder + .with_provenance( + &"id5-sync.com".parse()?, + EidProvenance::new() + .with_inserter("Lockr-For-Publishers") + .with_mm(2), + ) + .build() + } +} + +/// The Lockr capture provider. +pub struct LockrIdentityProvider { + descriptor: ProviderDescriptor, +} + +impl LockrIdentityProvider { + /// Builds the provider from Lockr's validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the descriptor is + /// invalid. + pub fn new(config: &LockrConfig) -> Result> { + Ok(Self { + descriptor: config.build_descriptor()?, + }) + } +} + +impl IdentityProvider for LockrIdentityProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn capture( + &self, + input: &CaptureInput<'_>, + ) -> Result> { + if input + .path() + .rsplit('/') + .next() + .is_some_and(|endpoint| endpoint == REVOKE_ENDPOINT) + { + return Ok(Resolution::opted_out()); + } + Ok(Resolution::no_match()) + } +} +``` + +`LockrConfig` already implements `IntegrationConfig`, so the `IdentityProviderConfig` impl is all the framework needs. The descriptor declares no `resolves`, which makes Lockr capture-only. Task 2 replaces the trailing `Ok(Resolution::no_match())` with token parsing. + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::lockr` +Expected: PASS, including the existing Lockr tests. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/ +git commit -S --signoff -m "Describe Lockr as a capture-only provider that opts out on revocation" +``` + +--- + +### Task 2: Read issued tokens + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/lockr_identity.rs` + +**Interfaces:** + +- Consumes: `CaptureInput::{response, source_named, now}`, `SourceDomain::matches`, `VendorEid::to_resolved`, `ResolvedEid::with_expires_at`, `UnixTime::from_millis`. +- Produces: the positive branch of `LockrIdentityProvider::capture`. + +- [ ] **Step 1: Write the failing tests** + +Add to the tests module: + +```rust + #[test] + fn reads_complete_eids_for_known_keys() { + let provider = provider(); + let fixture = CaptureFixture::new(Method::POST, &path("generate-tokens")).with_response( + StatusCode::OK, + &json!({ + "aimTokens": [{ "key_name": "id5id", "identity_expires": 1_770_000_000_000_u64 }], + "ids": { + "id5id": { "eid": { + "source": "id5-sync.com", + "uids": [{ "id": "ID5*example", "atype": 1 }], + "inserter": "inserter.example.com" + }}, + "unknown": { "eid": { + "source": "other.example.com", + "uids": [{ "id": "ignored" }] + }} + } + }), + ); + + let resolution = ProviderHarness::new(&provider) + .capture(&fixture) + .expect("should capture"); + + let eids = resolution.eids(); + assert_eq!(eids.len(), 1, "should keep only mapped keys"); + assert_eq!( + eids[0].source().as_str(), + "id5-sync.com", + "should map id5id" + ); + assert_eq!( + eids[0].inserter(), + Some("inserter.example.com"), + "should keep the EID fields" + ); + } + + #[test] + fn skips_an_expired_token() { + let provider = provider(); + let fixture = CaptureFixture::new(Method::POST, &path("refresh-tokens")).with_response( + StatusCode::OK, + &json!({ + "aimTokens": [{ "key_name": "id5id", "identity_expires": 1_000_u64 }], + "ids": { "id5id": { "eid": { + "source": "id5-sync.com", + "uids": [{ "id": "ID5*expired" }] + }}} + }), + ); + + let resolution = ProviderHarness::new(&provider) + .capture(&fixture) + .expect("should capture"); + + assert!(resolution.is_no_match(), "should skip the expired token"); + } +``` + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::lockr_identity` +Expected: `reads_complete_eids_for_known_keys` FAILS (the capture returns no match); `skips_an_expired_token` passes already and stays as a guard. + +- [ ] **Step 3: Add the reply types** + +Append after `impl LockrIdentityProvider`: + +```rust +#[derive(Deserialize)] +struct TokenReply { + #[serde(rename = "aimTokens", default)] + aim_tokens: Vec, + #[serde(default)] + ids: BTreeMap, +} + +#[derive(Deserialize)] +struct AimToken { + key_name: String, + /// Milliseconds since the epoch. + identity_expires: Option, +} + +#[derive(Deserialize)] +struct LockrId { + eid: VendorEid, +} +``` + +- [ ] **Step 4: Parse tokens in `capture`** + +In `capture`, replace the trailing `Ok(Resolution::no_match())` with: + +```rust + let Ok(reply) = input.response().json::() else { + return Ok(Resolution::no_match()); + }; + let eids: Vec = reply + .ids + .iter() + .filter_map(|(key, entry)| { + let (_, domain) = KEY_SOURCES.iter().find(|(name, _)| name == key)?; + let source = input.source_named(domain)?; + if !source.matches(&entry.eid.source) { + return None; + } + let expires = reply + .aim_tokens + .iter() + .find(|token| token.key_name == *key) + .and_then(|token| token.identity_expires) + .map(UnixTime::from_millis); + if expires.is_some_and(|at| at <= input.now()) { + return None; + } + let eid = entry.eid.to_resolved(source)?; + Some(match expires { + Some(at) => eid.with_expires_at(at), + None => eid, + }) + }) + .collect(); + Ok(if eids.is_empty() { + Resolution::no_match() + } else { + Resolution::found(eids) + }) +``` + +A body that is not a token reply (for example `page-view`'s) is no match, never an error: capture errors are reserved for provider bugs. `input.source_named` returns only sources the operator kept in `[ec.identity.providers.lockr].sources`. + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::lockr` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/lockr_identity.rs +git commit -S --signoff -m "Read Lockr-issued tokens so a new issue can clear a Lockr opt-out" +``` + +--- + +### Task 3: Register behind the flag, validate and document + +`lockr` is already in `builders()` and `DEPLOY_VALIDATED_INTEGRATION_IDS`. This task only adds the identity provider to its registration when the flag is on, and the descriptor to deploy validation. Lockr cannot use `validate_identity_integration`, which describes a provider whenever the integration is enabled: here the integration can be enabled without the provider. + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/lockr.rs` (`register`, tests) +- Modify: `crates/trusted-server-core/src/config.rs` (imports, `identity_descriptors_for_deploy`) +- Modify: `docs/guide/integrations/identity-vendors.md`, `CHANGELOG.md` + +**Interfaces:** + +- Consumes: `IntegrationRegistrationBuilder::try_with_identity_provider`, `IntegrationRegistration::identity_providers`. + +- [ ] **Step 1: Write the failing test** + +Add to `lockr.rs`'s tests module: + +```rust + #[test] + fn capture_identity_registers_an_identity_provider() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + LOCKR_INTEGRATION_ID, + &json!({ "enabled": true, "app_id": "app-1", "capture_identity": true }), + ) + .expect("should insert the Lockr config"); + + let registration = register(&settings) + .expect("should register") + .expect("should be enabled"); + + assert_eq!( + registration.identity_providers.len(), + 1, + "should register the capture provider" + ); + } +``` + +- [ ] **Step 2: Run it and see it fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::lockr::tests::capture_identity_registers_an_identity_provider` +Expected: FAIL, `identity_providers.len()` is 0. + +- [ ] **Step 3: Register the provider when the flag is on** + +In `lockr.rs`, import the provider: + +```rust +use crate::integrations::lockr_identity::LockrIdentityProvider; +``` + +and replace the end of `register`: + +```rust + let mut builder = IntegrationRegistration::builder(LOCKR_INTEGRATION_ID) + .with_proxy(integration.clone()) + .with_attribute_rewriter(integration.clone()); + if integration.config.capture_identity { + builder = + builder.try_with_identity_provider(LockrIdentityProvider::new(&integration.config))?; + } + Ok(Some(builder.build())) +``` + +- [ ] **Step 4: Describe it at deploy time** + +In `config.rs`, change the Lockr import to `lockr::{LOCKR_INTEGRATION_ID, LockrConfig}`, and in `identity_descriptors_for_deploy`, after the other identity integrations: + +```rust + if let Some(lockr) = settings.integration_config::(LOCKR_INTEGRATION_ID)? + && lockr.capture_identity + { + descriptors.push(lockr.build_descriptor().change_context( + TrustedServerError::Configuration { + message: format!( + "integrations.{LOCKR_INTEGRATION_ID}: the identity provider cannot be described" + ), + }, + )?); + } +``` + +Deploy validation then requires `[ec.identity.providers.lockr].sources` and an `[[ec.partners]]` entry for each listed source, as for any capture-only provider. + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::lockr` +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config` +Expected: PASS. + +- [ ] **Step 6: Document** + +In `docs/guide/integrations/identity-vendors.md`, add this table row: + +```markdown +| `lockr` | Lockr (capture) | Capture | `id5-sync.com`, `liveramp.com`, `uidapi.com` | Lockr's proxied API | +``` + +and this section: + +````markdown +## Lockr capture + +```toml +[integrations.lockr] +enabled = true +app_id = "app-1" +capture_identity = true + +[ec.identity.providers.lockr] +sources = ["id5-sync.com"] +``` + +With `capture_identity`, Lockr becomes a capture-only provider on the Lockr API calls Trusted Server already proxies. Capture is revoke-only: a `revoke-consent` call removes the IDs of the listed sources and blocks the browser's values for them until Lockr issues again. Lockr's IDs themselves reach KV through browser ingest. A capture-only provider must list its `sources`. +```` + +In `CHANGELOG.md` under `### Added`, add "revoke-only Lockr capture (`[integrations.lockr].capture_identity`)" to the vendor bullet, or, if there is no vendor bullet yet, add: + +```markdown +- Revoke-only Lockr capture (`[integrations.lockr].capture_identity`). Each source it lists needs an `[[ec.partners]]` entry. See [Identity Vendors](docs/guide/integrations/identity-vendors.md). +``` + +- [ ] **Step 7: Run the gates** + +Run: `cargo fmt --all -- --check && cargo test-fastly && cargo clippy-fastly` +Run: `cd docs && npm run format` +Run: `docs/node_modules/.bin/prettier --config docs/.prettierrc --check CHANGELOG.md` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add crates/trusted-server-core/src/ docs/guide/integrations/identity-vendors.md CHANGELOG.md +git commit -S --signoff -m "Register Lockr capture behind capture_identity and validate it at deploy" +``` + +## Self-Review Notes + +- Revoke-only, captured tokens never stored (the coordinator discards positive captures): Design notes, Task 2. +- Opt-out on `revoke-consent`: Task 1. Opt-out clear on a later issue: Task 2. +- Flag off changes nothing: `#[serde(default)]` and the `if` in Task 3. diff --git a/docs/superpowers/plans/2026-10-28-identity-provider-uid2.md b/docs/superpowers/plans/2026-10-28-identity-provider-uid2.md new file mode 100644 index 0000000000..928719356c --- /dev/null +++ b/docs/superpowers/plans/2026-10-28-identity-provider-uid2.md @@ -0,0 +1,903 @@ +# UID2 Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Generate and refresh UID2 tokens at the edge with a server-side `IdentityProvider` under `[integrations.uid2]`, writing the advertising token to the `uidapi.com` source. + +**Architecture:** `integrations/uid2.rs` holds `Uid2Config`, `Uid2Provider` and `register`. A first token comes from `POST /v2/token/generate` with the hashed email from `ts-hem`, sent in an AES-256-GCM envelope keyed by the operator's client secret. The refresh token and the key that decrypts its response are kept as private provider state, so later visits call `POST /v2/token/refresh` without the hashed email. The API key and client secret are secret fields: secret-store key names at rest, resolved values at runtime, never read by `build_descriptor`. + +**Tech Stack:** Rust 1.95 (edition 2024), `trusted-server-core`, `aes-gcm` 0.10 (new dependency), `base64`, `serde`/`serde_json`, `url`, `validator`, `error-stack`. + +**Spec:** `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`, section 12.3 (UID2: secrets, crypto, private state), section 10.2 (secret fields), and sections 4.2 to 4.6 (the provider API). Operator page: `docs/guide/integrations/identity-vendors.md`. + +**Prerequisite:** `docs/superpowers/plans/2026-10-28-identity-provider-inputs.md`. This plan needs `Input::HashedEmail` from the `ts-hem` cookie for the first token, and adds its section to the operator page that plan creates. + +## The UID2 wire format + +- **Generate request envelope:** base64 of `0x01 | IV (12 bytes) | AES-256-GCM(timestamp (8 bytes, big-endian ms) | nonce (8 bytes) | JSON payload)`, where the GCM output is ciphertext followed by its 16-byte tag, and the key is the base64-decoded 32-byte client secret. The payload is `{"email_hash": , "optout_check": 1}`. +- **Generate response:** base64 of `IV (12) | AES-256-GCM(timestamp (8) | nonce (8) | JSON)` under the same client key. The nonce must equal the request's nonce, or the response is rejected as malformed. The request nonce travels from `request` to `response` in the call context. +- **Refresh request:** the stored refresh token as the plain-text body. +- **Refresh response:** base64 of `IV (12) | AES-256-GCM(JSON)` under the stored `refresh_response_key`. The plaintext is JSON only, with no timestamp or nonce prefix. +- **Reply JSON:** `{"status": "...", "body": {advertising_token, refresh_token, refresh_response_key, identity_expires, refresh_from, refresh_expires}}`, times in milliseconds. `success` stores the token; `optout` opts the visitor out of UID2 and clears state; `expired_token` / `invalid_token` invalidate the stored token and clear state; any other status is malformed. +- Both calls send `Authorization: Bearer `. + +## Global Constraints + +- Follow `AGENTS.md`: `error-stack` with `derive_more::Display`, `expect("should ...")`, no local imports, comments above code, doc comment on every public item, Arrange-Act-Assert tests. +- Backtick `UID2` and other code-like names in doc comments (`clippy::doc_markdown`). +- Tests use fictional data: `uid2.example.com` base URL, fixed test keys (`[7; 32]`, `[9; 32]`), fake tokens. The `uidapi.com` source and the public default base URL may appear. +- Never log tokens, keys, the API key, the hashed email or decrypted payloads. Secrets are `Redacted`. +- `build_descriptor` must not read `api_key` or `client_secret` (deploy validation runs while they still hold secret-store key names). `check_descriptor_ignores_secrets` enforces it. +- Randomness comes only from `input.random_bytes()`, so the harness is deterministic. Time comes only from `input.now()`. +- UIDs use `atype` **3**. No TCF vendor by default; `tcf_vendor_id` adds one when the operator's UID2 agreement names it. +- Quick test: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::uid2`. Gates: `cargo test-fastly`, `cargo clippy-fastly`. + +## File Structure + +| File | Change | Responsibility | +| ----------------------------------------------------- | ------ | ------------------------------------------------------------ | +| `Cargo.toml`, `crates/trusted-server-core/Cargo.toml` | Modify | `aes-gcm = "0.10"` (workspace) and its use in core | +| `crates/trusted-server-core/src/integrations/uid2.rs` | Create | Config, crypto, provider, registration, tests | +| `crates/trusted-server-core/src/integrations/mod.rs` | Modify | `pub mod uid2;` and its `IntegrationBuilder` | +| `crates/trusted-server-core/src/config.rs` | Modify | Secret fields, runtime and deploy validation, deploy ID list | +| `docs/guide/integrations/identity-vendors.md` | Modify | Table row and `## UID2` section | +| `CHANGELOG.md` | Modify | Vendor bullet | + +--- + +### Task 1: Config, secrets and descriptor + +**Files:** + +- Modify: `Cargo.toml` (`[workspace.dependencies]`), `crates/trusted-server-core/Cargo.toml` (`[dependencies]`) +- Create: `crates/trusted-server-core/src/integrations/uid2.rs` +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (add `pub mod uid2;` in alphabetical order) +- Modify: `crates/trusted-server-core/src/config.rs` (`AppConfigMeta::secret_fields`, test `secret_metadata_lists_all_secret_paths_and_optionality`) + +**Interfaces:** + +- Produces: `pub(crate) const UID2_INTEGRATION_ID: &str = "uid2"`, `pub struct Uid2Config { enabled, base_url: String, api_key: Redacted, client_secret: Redacted, tcf_vendor_id: Option }`, `pub struct Uid2Provider { descriptor, api_key: Redacted, client_key: [u8; 32] }`, `pub fn Uid2Provider::new(&Uid2Config) -> Result>`. + +- [ ] **Step 1: Add the dependency** + +In the root `Cargo.toml` `[workspace.dependencies]`, in alphabetical order: + +```toml +aes-gcm = "0.10" +``` + +In `crates/trusted-server-core/Cargo.toml` `[dependencies]`: + +```toml +aes-gcm = { workspace = true } +``` + +`aes-gcm` is the RustCrypto AEAD implementation; it is pure Rust and builds for `wasm32-wasip1` and `wasm32-unknown-unknown`. + +- [ ] **Step 2: Write the failing tests** + +Create `uid2.rs` with the tests module: + +```rust +#[cfg(test)] +mod tests { + use http::StatusCode; + + use super::*; + use crate::ec::identity::test_support::{ + HarnessSetup, ProviderHarness, check_descriptor_ignores_secrets, + }; + use crate::ec::identity::{HashedEmail, SourceDomain, StoredId}; + + const CLIENT_KEY: [u8; 32] = [7; 32]; + const RESPONSE_KEY: [u8; 32] = [9; 32]; + + fn config(api_key: &str, client_secret: &str) -> Uid2Config { + Uid2Config { + enabled: true, + base_url: "https://uid2.example.com".to_owned(), + api_key: Redacted::new(api_key.to_owned()), + client_secret: Redacted::new(client_secret.to_owned()), + tcf_vendor_id: None, + } + } + + fn provider() -> Uid2Provider { + Uid2Provider::new(&config("api-key", &STANDARD.encode(CLIENT_KEY))) + .expect("should build the provider") + } + + #[test] + fn rejects_a_client_secret_that_is_not_a_key() { + assert!( + Uid2Provider::new(&config("api-key", "not-base64-key")).is_err(), + "should refuse an invalid client secret" + ); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets( + &config("api-key-name", "client-secret-name"), + &config("resolved-api-key", &STANDARD.encode(CLIENT_KEY)), + ); + } +} +``` + +In `config.rs`, in `secret_metadata_lists_all_secret_paths_and_optionality`, add after the `integrations.datadome.server_side_key_secret_name` entry: + +```rust + ("integrations.uid2.api_key".to_owned(), true), + ("integrations.uid2.client_secret".to_owned(), true), +``` + +- [ ] **Step 3: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::uid2` +Expected: FAIL to compile (`Uid2Config` not found). + +- [ ] **Step 4: Implement the config, descriptor and constructor** + +Prepend to `uid2.rs`. The `use` block is the module's final one; items used only by later tasks warn as unused until then. + +```rust +//! `UID2` identity provider: generates and refreshes `UID2` tokens server +//! side. +//! +//! A first token is generated from the hashed email in the `ts-hem` cookie, +//! sent in an AES-256-GCM envelope keyed by the operator's client secret. +//! The refresh token and the key that decrypts its response are kept as +//! private state, so later visits refresh without the hashed email. + +use aes_gcm::aead::{Aead as _, KeyInit as _}; +use aes_gcm::{Aes256Gcm, Nonce}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use error_stack::Report; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use url::Url; +use validator::Validate; + +use crate::ec::identity::{ + ConsentPolicy, IdentityProvider, IdentityProviderConfig, IdentityProviderError, Input, + ProviderDescriptor, ProviderRequest, ProviderResponse, Resolution, ResolveInput, ResolveKind, + ResolvedEid, ResolvedUid, Resolves, UnixTime, +}; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistration; +use crate::redacted::Redacted; +use crate::settings::{IntegrationConfig, Settings}; + +/// Integration ID of the `UID2` provider. +pub(crate) const UID2_INTEGRATION_ID: &str = "uid2"; + +/// The envelope version byte. +const ENVELOPE_VERSION: u8 = 1; +const IV_BYTES: usize = 12; +const TAG_BYTES: usize = 16; +const TIMESTAMP_BYTES: usize = 8; +const NONCE_BYTES: usize = 8; + +/// Config of the `uid2` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct Uid2Config { + /// Enables the provider. + #[serde(default)] + pub enabled: bool, + /// The operator base URL, for example `https://prod.uidapi.com`. + #[serde(default = "default_base_url")] + #[validate(url)] + pub base_url: String, + /// The API key. A secret-store key name at rest, the resolved value at + /// runtime. + pub api_key: Redacted, + /// The base64 client secret that keys request envelopes. A secret-store + /// key name at rest, the resolved value at runtime. + pub client_secret: Redacted, + /// The IAB Global Vendor List ID whose consent `UID2` calls require where + /// TCF applies, if any. + pub tcf_vendor_id: Option, +} + +fn default_base_url() -> String { + "https://prod.uidapi.com".to_owned() +} + +impl IntegrationConfig for Uid2Config { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl IdentityProviderConfig for Uid2Config { + fn build_descriptor(&self) -> Result> { + let mut consent = ConsentPolicy::strict(); + if let Some(vendor) = self.tcf_vendor_id { + consent = consent.with_tcf_vendor(vendor); + } + ProviderDescriptor::builder(UID2_INTEGRATION_ID.parse()?) + .with_source("uidapi.com".parse()?) + .with_expiring_ids() + .with_resolves( + Resolves::new() + .with_endpoint(&self.base_url) + .with_needs_for(ResolveKind::Initial, [Input::HashedEmail]) + .with_private_state(), + ) + .with_consent_policy(consent) + .build() + } +} + +/// The `UID2` provider. +pub struct Uid2Provider { + descriptor: ProviderDescriptor, + api_key: Redacted, + client_key: [u8; 32], +} + +impl Uid2Provider { + /// Builds the provider from its config with resolved secrets. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the client secret is + /// not 32 base64 bytes, or the config cannot describe a valid provider. + pub fn new(config: &Uid2Config) -> Result> { + let client_key = STANDARD + .decode(config.client_secret.expose().trim()) + .ok() + .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok()) + .ok_or_else(|| { + IdentityProviderError::config("`client_secret` is not 32 base64-encoded bytes") + })?; + Ok(Self { + descriptor: config.build_descriptor()?, + api_key: config.api_key.clone(), + client_key, + }) + } +} + +impl IdentityProvider for Uid2Provider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } +} +``` + +The hashed email is needed only for `Initial`; a refresh uses private state. Decoding the client secret happens in `new`, at registration, where secrets are resolved, so a wrong secret fails startup instead of every call. + +- [ ] **Step 5: Declare the secret fields** + +In `config.rs`, in `TrustedServerAppConfig`'s `secret_fields()` list, after the DataDome `server_side_key_secret_name` field: + +```rust + field( + vec![ + optional_object("integrations"), + optional_object("uid2"), + object("api_key"), + ], + true, + ), + field( + vec![ + optional_object("integrations"), + optional_object("uid2"), + object("client_secret"), + ], + true, + ), +``` + +`ts config push` then resolves both from the secret store, and the serialized app config keeps only their key names. + +- [ ] **Step 6: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::uid2` +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config::tests::secret_metadata_lists_all_secret_paths_and_optionality` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add Cargo.toml Cargo.lock crates/trusted-server-core/Cargo.toml crates/trusted-server-core/src/integrations/ crates/trusted-server-core/src/config.rs +git commit -S --signoff -m "Add UID2 provider config with secret-store API key and client secret" +``` + +--- + +### Task 2: Generate a token + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/uid2.rs` + +**Interfaces:** + +- Consumes: `ResolveInput::{endpoint, random_bytes, now, hashed_email, primary_source}`, `HashedEmail::to_base64`, `ProviderRequest::{post_text, with_bearer, try_with_call_context}`, `ProviderResponse::{success_text, call_context}`, `UnixTime::from_millis`. +- Produces: `fn seal(&[u8; 32], &[u8; 12], &[u8]) -> Result, _>`, `fn open(&[u8], &str) -> Result, _>`, `fn encrypt_envelope(..) -> Result`, `fn decrypt_generate(&[u8; 32], &str, &[u8; 8]) -> Result`, `pub(crate) struct Uid2State { refresh_token, refresh_response_key, refresh_expires: u64 }`, `struct GenerateCall { request_nonce: [u8; 8] }`. + +- [ ] **Step 1: Write the failing tests** + +Add to the tests module: + +```rust + fn sealed_response(key: &[u8; 32], plaintext: &[u8]) -> String { + let iv = [3u8; IV_BYTES]; + let mut bytes = iv.to_vec(); + bytes.extend(seal(key, &iv, plaintext).expect("should seal")); + STANDARD.encode(bytes) + } + + fn identity() -> serde_json::Value { + json!({ + "status": "success", + "body": { + "advertising_token": "advertising-token", + "refresh_token": "refresh-token", + "refresh_response_key": STANDARD.encode(RESPONSE_KEY), + "identity_expires": 1_760_003_600_000_u64, + "refresh_from": 1_760_001_800_000_u64, + "refresh_expires": 1_762_592_000_000_u64 + } + }) + } + + #[test] + fn generates_a_token_from_the_hashed_email() { + let provider = provider(); + let mut harness = + ProviderHarness::new(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + assert_eq!( + request.url().as_str(), + "https://uid2.example.com/v2/token/generate", + "should call token/generate" + ); + let envelope = STANDARD + .decode(request.body()) + .expect("should send a base64 envelope"); + assert_eq!( + envelope[0], ENVELOPE_VERSION, + "should lead with the version" + ); + let plaintext = Aes256Gcm::new((&CLIENT_KEY).into()) + .decrypt( + Nonce::from_slice(&envelope[1..=IV_BYTES]), + &envelope[1 + IV_BYTES..], + ) + .expect("should open with the client secret"); + let nonce: [u8; NONCE_BYTES] = plaintext[TIMESTAMP_BYTES..TIMESTAMP_BYTES + NONCE_BYTES] + .try_into() + .expect("should hold a nonce"); + let payload: serde_json::Value = + serde_json::from_slice(&plaintext[TIMESTAMP_BYTES + NONCE_BYTES..]) + .expect("should hold JSON"); + assert_eq!( + payload["email_hash"], + STANDARD.encode([0xab; 32]), + "should send the base64 hash" + ); + + let mut reply = vec![0u8; TIMESTAMP_BYTES]; + reply.extend_from_slice(&nonce); + reply.extend(identity().to_string().into_bytes()); + let resolution = harness + .respond_raw( + StatusCode::OK, + &[], + sealed_response(&CLIENT_KEY, &reply).as_bytes(), + ) + .expect("should decrypt the response"); + + assert_eq!( + resolution.eids()[0].uids()[0].id(), + "advertising-token", + "should store the advertising token" + ); + let state: Uid2State = harness.state_of(&resolution).expect("should keep state"); + assert_eq!( + state.refresh_token, "refresh-token", + "should keep the refresh token" + ); + } + + #[test] + fn rejects_a_generate_response_with_another_nonce() { + let provider = provider(); + let mut harness = + ProviderHarness::new(&provider).with_hashed_email(HashedEmail::for_test([0xab; 32])); + harness.request().expect("should build a request"); + + let mut reply = vec![0u8; TIMESTAMP_BYTES + NONCE_BYTES]; + reply.extend(identity().to_string().into_bytes()); + let error = harness + .respond_raw( + StatusCode::OK, + &[], + sealed_response(&CLIENT_KEY, &reply).as_bytes(), + ) + .expect_err("should refuse a mismatched nonce"); + + assert!( + matches!(error.current_context(), IdentityProviderError::Malformed), + "should be malformed" + ); + } +``` + +The first test decrypts the request with the AEAD directly, independent of the module's helpers, so it checks the wire format rather than round-tripping the provider's own code. The second seals an all-zero nonce, which differs from the nonce the seeded harness randomness gives the request. + +- [ ] **Step 2: Run them and see them fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::uid2` +Expected: FAIL to compile (`seal`, `Uid2State` not found). + +- [ ] **Step 3: Add state, call context and reply types** + +Append after `impl Uid2Provider`: + +```rust +/// The refresh anchor kept between calls. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct Uid2State { + refresh_token: String, + refresh_response_key: String, + /// Milliseconds since the epoch. + refresh_expires: u64, +} + +/// The nonce a generate call sent, which its response must echo. +#[derive(Serialize, Deserialize)] +struct GenerateCall { + request_nonce: [u8; NONCE_BYTES], +} + +#[derive(Deserialize)] +struct Uid2Reply { + status: String, + body: Option, +} + +#[derive(Deserialize)] +struct Uid2Identity { + advertising_token: String, + refresh_token: String, + refresh_response_key: String, + /// Milliseconds since the epoch, like the two below. + identity_expires: u64, + refresh_from: u64, + refresh_expires: u64, +} +``` + +The state (refresh token, response key and expiry) fits within the framework's 1,024-byte private-state cap. + +- [ ] **Step 4: Add the crypto helpers** + +Append: + +```rust +/// `path` under `base`, keeping a `base` path that has no trailing slash. +fn join(base: &Url, path: &str) -> Result> { + let mut base = base.clone(); + if !base.path().ends_with('/') { + let directory = format!("{}/", base.path()); + base.set_path(&directory); + } + base.join(path) + .map_err(|_| IdentityProviderError::config("`base_url` cannot take a path")) +} + +/// AES-256-GCM of `plaintext` under `key`, as `ciphertext || tag`. +fn seal( + key: &[u8; 32], + iv: &[u8; IV_BYTES], + plaintext: &[u8], +) -> Result, Report> { + Aes256Gcm::new(key.into()) + .encrypt(Nonce::from_slice(iv), plaintext) + .map_err(|_| IdentityProviderError::config("envelope encryption failed")) +} + +/// Opens base64 `IV || ciphertext || tag` under `key`. +fn open(key: &[u8], body: &str) -> Result, Report> { + let bytes = STANDARD + .decode(body.trim()) + .map_err(|_| IdentityProviderError::malformed())?; + if bytes.len() < IV_BYTES + TAG_BYTES { + return Err(IdentityProviderError::malformed()); + } + let cipher = Aes256Gcm::new_from_slice(key).map_err(|_| IdentityProviderError::malformed())?; + let (iv, sealed) = bytes.split_at(IV_BYTES); + cipher + .decrypt(Nonce::from_slice(iv), sealed) + .map_err(|_| IdentityProviderError::malformed()) +} + +/// The token-generate envelope: base64 of the version byte, the IV, and the +/// sealed timestamp, nonce and JSON payload. +fn encrypt_envelope( + key: &[u8; 32], + iv: &[u8; IV_BYTES], + now: UnixTime, + nonce: &[u8; NONCE_BYTES], + payload: &serde_json::Value, +) -> Result> { + let mut plaintext = Vec::with_capacity(TIMESTAMP_BYTES + NONCE_BYTES + 64); + plaintext.extend_from_slice(&now.as_secs().saturating_mul(1000).to_be_bytes()); + plaintext.extend_from_slice(nonce); + plaintext.extend_from_slice(payload.to_string().as_bytes()); + let sealed = seal(key, iv, &plaintext)?; + let mut envelope = Vec::with_capacity(1 + IV_BYTES + sealed.len()); + envelope.push(ENVELOPE_VERSION); + envelope.extend_from_slice(iv); + envelope.extend_from_slice(&sealed); + Ok(STANDARD.encode(envelope)) +} + +/// Opens a generate response and checks that it echoes `nonce`. +fn decrypt_generate( + key: &[u8; 32], + body: &str, + nonce: &[u8; NONCE_BYTES], +) -> Result> { + let plaintext = open(key, body)?; + let payload = plaintext + .get(TIMESTAMP_BYTES..) + .and_then(|rest| rest.strip_prefix(nonce.as_slice())) + .ok_or_else(IdentityProviderError::malformed)?; + serde_json::from_slice(payload).map_err(|_| IdentityProviderError::malformed()) +} +``` + +A short body, a bad tag, or a nonce mismatch are all `Malformed`, which the coordinator backs off like any unparseable response. `join` with a relative path keeps a base URL's own path prefix only when it ends in `/`; the default has none. + +- [ ] **Step 5: Implement the generate request and response** + +Add to `impl IdentityProvider for Uid2Provider`: + +```rust + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let base = input.endpoint()?; + let url = join(base, "v2/token/generate")?; + let request_nonce = input.random_bytes::(); + let envelope = encrypt_envelope( + &self.client_key, + &input.random_bytes::(), + input.now(), + &request_nonce, + &json!({ "email_hash": input.hashed_email()?.to_base64(), "optout_check": 1 }), + )?; + Ok(Some( + ProviderRequest::post_text(&url, &envelope) + .with_bearer(&self.api_key) + .try_with_call_context(&GenerateCall { request_nonce })?, + )) + } + + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let body = response.success_text()?; + let call = response + .call_context::()? + .ok_or_else(|| IdentityProviderError::config("missing call context"))?; + let reply = decrypt_generate(&self.client_key, body, &call.request_nonce)?; + match reply.status.as_str() { + "optout" => Ok(Resolution::opted_out().clearing_state()), + "expired_token" | "invalid_token" => { + Ok(Resolution::invalidate(input.primary_source()).clearing_state()) + } + "success" => { + let identity = reply.body.ok_or_else(IdentityProviderError::malformed)?; + let uid = ResolvedUid::new(identity.advertising_token).with_atype(3); + let eid = ResolvedEid::new(input.primary_source(), uid) + .with_expires_at(UnixTime::from_millis(identity.identity_expires)) + .with_refresh_at(UnixTime::from_millis(identity.refresh_from)) + .with_refresh_until(UnixTime::from_millis(identity.refresh_expires)); + Ok(Resolution::found([eid]).replacing_state(&Uid2State { + refresh_token: identity.refresh_token, + refresh_response_key: identity.refresh_response_key, + refresh_expires: identity.refresh_expires, + })) + } + _ => Err(IdentityProviderError::malformed()), + } + } +``` + +The token's own times drive the framework: `identity_expires` is the expiry, `refresh_from` the earliest refresh, and `refresh_expires` the last moment a refresh can succeed. Task 3 adds the refresh call and replaces the call-context line in `response`. + +- [ ] **Step 6: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::uid2` +Expected: PASS (4 tests). + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/uid2.rs +git commit -S --signoff -m "Generate UID2 tokens through an encrypted envelope" +``` + +--- + +### Task 3: Refresh with the stored state + +A refresh is chosen when the framework schedules a `Refresh` for `uidapi.com`. It needs the stored state; without state, or once `refresh_expires` has passed, the provider skips the call (returns `None`), and the next initial lookup with a hashed email starts over. A refresh call carries no call context, which is how `response` tells the two apart. + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/uid2.rs` + +**Interfaces:** + +- Consumes: `ResolveInput::{is_refresh, state}`, `Uid2State`, `open`. +- Produces: `fn decrypt_refresh(key: &str, body: &str) -> Result>`. + +- [ ] **Step 1: Write the failing test** + +Add to the tests module: + +```rust + #[test] + fn refreshes_with_the_stored_state() { + let provider = provider(); + let source: SourceDomain = "uidapi.com".parse().expect("should parse the source"); + let mut harness = ProviderHarness::new(&provider).with_setup( + HarnessSetup::default() + .with_stored( + StoredId::for_test(&source, "old-token").expiring_at(UnixTime::from_secs(1)), + ) + .with_state(&Uid2State { + refresh_token: "stored-refresh".to_owned(), + refresh_response_key: STANDARD.encode(RESPONSE_KEY), + refresh_expires: 1_770_000_000_000, + }), + ); + + let request = harness + .request() + .expect("should build a request") + .expect("should not skip"); + assert_eq!( + request.body(), + b"stored-refresh", + "should send the refresh token" + ); + + let resolution = harness + .respond_raw( + StatusCode::OK, + &[], + sealed_response( + &RESPONSE_KEY, + json!({ "status": "optout" }).to_string().as_bytes(), + ) + .as_bytes(), + ) + .expect("should decrypt the refresh response"); + assert!(resolution.is_opted_out(), "should report the opt-out"); + } +``` + +The refresh response plaintext is JSON only, unlike the generate response, which is why `sealed_response` here seals bare JSON. + +- [ ] **Step 2: Run it and see it fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::uid2::tests::refreshes_with_the_stored_state` +Expected: FAIL. Without a refresh branch the provider takes the generate path, which fails on the missing hashed email. + +- [ ] **Step 3: Add the refresh request** + +At the top of `request`, after `let base = input.endpoint()?;`, insert: + +```rust + if input.is_refresh(input.primary_source()) { + let Some(state) = input.state::()? else { + return Ok(None); + }; + if UnixTime::from_millis(state.refresh_expires) <= input.now() { + return Ok(None); + } + let url = join(base, "v2/token/refresh")?; + return Ok(Some( + ProviderRequest::post_text(&url, &state.refresh_token).with_bearer(&self.api_key), + )); + } +``` + +- [ ] **Step 4: Decrypt refresh responses** + +Append after `decrypt_generate`: + +```rust +/// Opens a refresh response with the stored base64 response key. +fn decrypt_refresh(key: &str, body: &str) -> Result> { + let key = STANDARD + .decode(key) + .map_err(|_| IdentityProviderError::malformed())?; + let plaintext = open(&key, body)?; + serde_json::from_slice(&plaintext).map_err(|_| IdentityProviderError::malformed()) +} +``` + +In `response`, replace + +```rust + let call = response + .call_context::()? + .ok_or_else(|| IdentityProviderError::config("missing call context"))?; + let reply = decrypt_generate(&self.client_key, body, &call.request_nonce)?; +``` + +with + +```rust + let reply = match response.call_context::()? { + Some(call) => decrypt_generate(&self.client_key, body, &call.request_nonce)?, + None => { + let state = input + .state::()? + .ok_or_else(|| IdentityProviderError::config("refresh without state"))?; + decrypt_refresh(&state.refresh_response_key, body)? + } + }; +``` + +A successful refresh goes through the same `success` arm and replaces the state with the new refresh token and response key. + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib integrations::uid2` +Expected: PASS (5 tests). + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/integrations/uid2.rs +git commit -S --signoff -m "Refresh UID2 tokens from the stored refresh state" +``` + +--- + +### Task 4: Register, validate and document + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/uid2.rs` (`register`) +- Modify: `crates/trusted-server-core/src/integrations/mod.rs` (`builders()`) +- Modify: `crates/trusted-server-core/src/config.rs` +- Modify: `docs/guide/integrations/identity-vendors.md`, `CHANGELOG.md` + +**Interfaces:** + +- Produces: `pub fn register(settings: &Settings) -> Result, Report>`. + +- [ ] **Step 1: Write `register` and add the builder** + +Append to `uid2.rs`, before the tests module: + +```rust +/// Registers the `UID2` provider when `[integrations.uid2]` is enabled. +/// +/// # Errors +/// +/// Returns an error when the config does not parse, the client secret is +/// invalid, or the config describes an invalid provider. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.integration_config::(UID2_INTEGRATION_ID)? else { + return Ok(None); + }; + log::info!("Registering UID2 identity provider"); + Ok(Some( + IntegrationRegistration::builder(UID2_INTEGRATION_ID) + .try_with_identity_provider(Uid2Provider::new(&config))? + .without_js() + .build(), + )) +} +``` + +In `integrations/mod.rs` `builders()`, add: + +```rust + IntegrationBuilder { + id: uid2::UID2_INTEGRATION_ID, + build: uid2::register, + }, +``` + +- [ ] **Step 2: Run the deploy-coverage test and see it fail** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config::tests::deploy_validation_covers_registered_integration_builders` +Expected: FAIL with `missing_ids` containing `"uid2"`. + +- [ ] **Step 3: Validate at runtime and at deploy** + +In `config.rs`, import `uid2::{UID2_INTEGRATION_ID, Uid2Config}`. Add `UID2_INTEGRATION_ID,` to `DEPLOY_VALIDATED_INTEGRATION_IDS`. In `identity_descriptors_for_deploy`: + +```rust + descriptors.extend(validate_identity_integration::( + settings, + UID2_INTEGRATION_ID, + )?); +``` + +`validate_identity_integration` only calls `build_descriptor`, never `Uid2Provider::new`, so deploy validation works while the secrets are still key names. If it still carries the framework's temporary `#[allow(dead_code)]`, remove it and its comment. In `validate_enabled_integrations`: + +```rust + validate_integration::(settings, UID2_INTEGRATION_ID)?; +``` + +- [ ] **Step 4: Run the tests** + +Run: `cargo test -p trusted-server-core --features test-utils --target x86_64-unknown-linux-gnu --lib config` +Expected: PASS. + +- [ ] **Step 5: Document** + +In `docs/guide/integrations/identity-vendors.md`, add this table row: + +```markdown +| `uid2` | UID2 | Resolve | `uidapi.com` | Hashed email for a first token | +``` + +and this section: + +````markdown +## UID2 + +```toml +[integrations.uid2] +enabled = true +# base_url = "https://prod.uidapi.com" +api_key = "uid2_api_key" +client_secret = "uid2_client_secret" +# tcf_vendor_id = 21 +``` + +`api_key` and `client_secret` are secret-store key names; `ts config push` resolves them. The client secret is the base64 key that encrypts token requests. A first token needs the hashed email. Trusted Server keeps the refresh token as private state and refreshes before the token expires. A UID2 `optout` opts the visitor out of UID2 until they consent again. UID2 is not meant for the EU (EUID covers it), so set `tcf_vendor_id` only when your UID2 agreement names one. +```` + +In `CHANGELOG.md` under `### Added`, insert "UID2 (`[integrations.uid2]`)" into the vendor bullet's list of providers, or, if there is no vendor bullet yet, add: + +```markdown +- Server-side identity provider for UID2 (`[integrations.uid2]`). Each source it writes needs an `[[ec.partners]]` entry. See [Identity Vendors](docs/guide/integrations/identity-vendors.md). +``` + +- [ ] **Step 6: Run the gates** + +Run: `cargo fmt --all -- --check && cargo test-fastly && cargo clippy-fastly && cargo clippy-cloudflare-wasm && cargo clippy-spin-wasm` +Run: `cd docs && npm run format` +Run: `docs/node_modules/.bin/prettier --config docs/.prettierrc --check CHANGELOG.md` +Expected: PASS. The two extra wasm clippy runs confirm `aes-gcm` builds for every wasm adapter. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/ docs/guide/integrations/identity-vendors.md CHANGELOG.md +git commit -S --signoff -m "Register the UID2 identity provider and validate its config" +``` + +## Self-Review Notes + +- Generate envelope `0x01 | IV12 | GCM(ts8 | nonce8 | JSON)`, nonce echo check: Task 2. +- Refresh decrypts with `refresh_response_key`, JSON-only plaintext: Task 3. +- Secrets in `secret_fields()`, never read by `build_descriptor`: Tasks 1 and 4.