diff --git a/Cargo.toml b/Cargo.toml index 2d56af4bb2..c0d79bc033 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ rustls = "0.23" rustls-pemfile = "2" scraper = "0.24.0" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0.149" +serde_json = { version = "1.0.149", features = ["raw_value"] } serde_yaml_ng = "0.10" sha2 = "0.10.9" similar = "2.7" diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 1d58c229e4..bf09c2086e 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -2297,6 +2297,38 @@ mod tests { .eid_sync_source() } + #[test] + fn auction_route_carries_the_raw_ts_eids_cookie_to_finalization() { + let router = test_router(); + let eids_cookie = base64::engine::general_purpose::STANDARD.encode( + serde_json::json!([{ "source": "ids.example.com", "uids": [{ "id": "example-uid" }] }]) + .to_string(), + ); + let mut request = browser_request(Method::POST, "/auction", "empty"); + request.headers_mut().insert( + header::COOKIE, + HeaderValue::from_str(&format!("ts-eids={eids_cookie}")) + .expect("should build cookie header"), + ); + + let response = route(&router, request); + + let state = response + .extensions() + .get::() + .expect("auction responses should carry EC finalization state"); + assert_eq!( + state.eids_cookie.as_deref(), + Some(eids_cookie.as_str()), + "the adapter should hand finalization the raw ts-eids cookie" + ); + assert_eq!( + state.ec_context.eid_sync_source(), + Some(EidSyncSource::Auction), + "finalization should know this was an /auction request" + ); + } + #[test] fn dispatch_limits_returning_user_eid_sync_to_eligible_routes() { let router = test_router(); diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index f364464561..c21caa1b0a 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -1,11 +1,11 @@ //! HTTP endpoint handlers for auction requests. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{Request, Response, StatusCode, header}; -use serde_json::Value as JsonValue; +use serde_json::value::RawValue; use crate::auction::formats::AdRequest; use crate::auction::orchestrator::OrchestrationResult; @@ -14,13 +14,14 @@ use crate::constants::COOKIE_TS_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; use crate::ec::EcKvSnapshot; +use crate::ec::client_eids::parse_client_eids; use crate::ec::eids::{resolve_partner_ids, to_eids}; use crate::ec::kv::KvIdentityGraph; -use crate::ec::kv_types::MAX_UID_LENGTH; +use crate::ec::partner::normalize_partner_source_domain; use crate::ec::prebid_eids::parse_prebid_eids_cookie; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; -use crate::openrtb::{Eid, Uid}; +use crate::openrtb::Eid; use crate::platform::RuntimeServices; use crate::settings::Settings; @@ -35,10 +36,6 @@ use super::telemetry::{ }; use super::types::AuctionContext; -const MAX_CLIENT_EID_SOURCES: usize = 64; -const MAX_CLIENT_UIDS_PER_SOURCE: usize = 32; -const MAX_CLIENT_EID_SOURCE_BYTES: usize = 255; - /// Maximum accepted JSON body size for `/auction`. Picked to comfortably fit /// the largest realistic Prebid-derived auction request (hundreds of ad units /// with EID arrays) while preventing an authenticated client from consuming @@ -292,7 +289,7 @@ pub async fn handle_auction( // `ec_id.is_some()`. let client_eids = if ec_id.is_some() { resolve_client_auction_eids( - body.eids.as_ref(), + body.eids.as_deref(), extract_cookie_value(&http_req, COOKIE_TS_EIDS).as_deref(), ) } else { @@ -342,8 +339,8 @@ pub async fn handle_auction( geo, )?; - // Merge current-request client EIDs with KV-resolved EIDs, then apply - // consent gating before attaching them to the auction request. + // Forward the browser entries as sent, append KV entries for registered + // sources the browser did not send, then apply consent gating. let merged_eids = merge_auction_eids(client_eids, eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); auction_request.user.eids = @@ -469,11 +466,18 @@ pub(crate) fn resolve_auction_eids( Some(to_eids(&resolved)) } +/// Resolves the request's browser EIDs: the `/auction` body when it has a +/// valid entry, else the `ts-eids` cookie. +/// +/// Both go through the shared client EID parser, so the same rules and caps +/// apply to each, and a malformed entry costs only itself. pub(crate) fn resolve_client_auction_eids( - raw: Option<&JsonValue>, + raw: Option<&RawValue>, cookie_value: Option<&str>, ) -> Option> { - parse_client_auction_eids(raw).or_else(|| parse_cookie_auction_eids(cookie_value)) + raw.map(parse_client_eids) + .filter(|eids| !eids.is_empty()) + .or_else(|| parse_cookie_auction_eids(cookie_value)) } fn parse_cookie_auction_eids(cookie_value: Option<&str>) -> Option> { @@ -488,132 +492,41 @@ fn parse_cookie_auction_eids(cookie_value: Option<&str>) -> Option> { } } -fn parse_client_auction_eids(raw: Option<&JsonValue>) -> Option> { - let Some(JsonValue::Array(entries)) = raw else { - return None; - }; - - let mut eids = Vec::new(); - - for entry in entries { - if eids.len() >= MAX_CLIENT_EID_SOURCES { - log::debug!( - "Auction EIDs: reached max client EID source count ({MAX_CLIENT_EID_SOURCES})" - ); - break; - } - let JsonValue::Object(entry) = entry else { - log::debug!("Auction EIDs: dropping malformed client EID entry"); - continue; - }; - - let Some(source) = entry - .get("source") - .and_then(JsonValue::as_str) - .filter(|source| !source.trim().is_empty()) - .filter(|source| source.len() <= MAX_CLIENT_EID_SOURCE_BYTES) - .map(str::to_owned) - else { - continue; - }; - - let Some(JsonValue::Array(raw_uids)) = entry.get("uids") else { - continue; - }; - - let uids: Vec<_> = raw_uids - .iter() - .filter_map(parse_client_auction_uid) - .take(MAX_CLIENT_UIDS_PER_SOURCE) - .collect(); - if uids.is_empty() { - continue; - } - - eids.push(Eid { source, uids }); - } - - if eids.is_empty() { None } else { Some(eids) } -} - -fn parse_client_auction_uid(raw: &JsonValue) -> Option { - let JsonValue::Object(uid) = raw else { - return None; - }; - - let id = uid - .get("id") - .and_then(JsonValue::as_str) - .filter(|id| !id.trim().is_empty()) - .filter(|id| id.len() <= MAX_UID_LENGTH)? - .to_owned(); - - let atype = uid - .get("atype") - .and_then(JsonValue::as_u64) - .and_then(|atype| i32::try_from(atype).ok()); - - let ext = match uid.get("ext") { - Some(JsonValue::Object(_)) => uid.get("ext").cloned(), - _ => None, - }; - - Some(Uid { id, atype, ext }) -} - +/// Merges the request's browser EIDs with KV-resolved partner EIDs. +/// +/// Browser entries are forwarded as parsed, in wire order, never altered or +/// combined: they are what Prebid sent the `trustedServer` adapter, and a +/// source may repeat with different EID-level fields. KV entries are appended +/// after them, only for registered sources no browser entry carries. Sources +/// are compared as the partner registry normalizes them (lowercase, trailing +/// dot removed). For a source the browser sends, KV's UIDs, `atype` and `ext` +/// are not used: KV keeps the first value it stored, so it is the staler copy. +/// Entries without UIDs are dropped. pub(crate) fn merge_auction_eids( client_eids: Option>, resolved_eids: Option>, ) -> Option> { - let mut merged = Vec::new(); - - for eid in resolved_eids + let mut merged: Vec = client_eids .into_iter() .flatten() - .chain(client_eids.into_iter().flatten()) - { - if eid.source.is_empty() { + .filter(|eid| !eid.uids.is_empty()) + .collect(); + let browser_sources: HashSet = merged + .iter() + .filter_map(|eid| normalize_partner_source_domain(&eid.source).ok()) + .collect(); + + for eid in resolved_eids.into_iter().flatten() { + if eid.uids.is_empty() { continue; } - - let source_index = match merged - .iter() - .position(|existing: &Eid| existing.source == eid.source) - { - Some(index) => index, - None => { - merged.push(Eid { - source: eid.source.clone(), - uids: Vec::new(), - }); - merged.len() - 1 - } - }; - - for uid in eid.uids { - if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { - continue; - } - - if let Some(existing_uid) = merged[source_index] - .uids - .iter_mut() - .find(|existing| existing.id == uid.id) - { - if existing_uid.atype.is_none() { - existing_uid.atype = uid.atype; - } - if existing_uid.ext.is_none() { - existing_uid.ext = uid.ext; - } - } else { - merged[source_index].uids.push(uid); - } + let carried_by_browser = normalize_partner_source_domain(&eid.source) + .is_ok_and(|source| browser_sources.contains(&source)); + if !carried_by_browser { + merged.push(eid); } } - merged.retain(|eid| !eid.uids.is_empty()); - if merged.is_empty() { None } else { @@ -640,6 +553,7 @@ mod tests { use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; + use serde_json::Value as JsonValue; use serde_json::json; use std::sync::{Arc, Mutex}; @@ -803,6 +717,183 @@ mod tests { ); } + const COOKIE_ONLY_SOURCE: &str = "cookie-only.example.com"; + + fn ingestion_registry(extra_source: &str) -> PartnerRegistry { + PartnerRegistry::from_config(&[ + counting_test_partner(COOKIE_ONLY_SOURCE), + counting_test_partner(extra_source), + ]) + .expect("should build partner registry") + } + + fn cookie_only_ts_eids() -> String { + BASE64.encode( + serde_json::to_vec(&json!([ + { "source": COOKIE_ONLY_SOURCE, "uids": [{ "id": "cookie-uid" }] } + ])) + .expect("should serialize cookie"), + ) + } + + fn seeded_graph(ec_id: &str) -> KvIdentityGraph { + let graph = KvIdentityGraph::in_memory("test_store"); + let mut live = crate::ec::kv_types::KvEntry::tombstone(1000); + live.consent.ok = true; + graph.create(ec_id, &live).expect("should seed live row"); + graph + } + + fn stored_ids(graph: &KvIdentityGraph, ec_id: &str) -> Vec<(String, String)> { + let (stored, _) = graph + .get(ec_id) + .expect("should read store") + .expect("row should exist"); + stored + .ids + .iter() + .map(|(source, id)| (source.clone(), id.uid.clone())) + .collect() + } + + #[tokio::test] + async fn auction_ingests_its_body_eids_and_not_a_cookie_only_source() { + let settings = create_test_settings(); + let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + timeout_ms: 2000, + mediator: None, + ..Default::default() + }); + orchestrator.register_provider(Arc::new(EidCapturingProvider { + had_eids: Arc::new(std::sync::Mutex::new(None)), + })); + let registry = ingestion_registry("body.example.com"); + let ec_id = format!("{}.ingbdy", "a".repeat(64)); + let graph = seeded_graph(&ec_id); + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + ec_context.set_eid_sync_source(crate::ec::EidSyncSource::Auction); + let cookie = cookie_only_ts_eids(); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .header(header::COOKIE, format!("{COOKIE_TS_EIDS}={cookie}")) + .body(EdgeBody::from( + serde_json::to_vec(&json!({ + "adUnits": [{ "code": "div-gpt-ad-1", "mediaTypes": { "banner": { "sizes": [[300, 250]] } } }], + "eids": [{ "source": "body.example.com", "uids": [{ "id": "body-uid" }] }] + })) + .expect("should serialize body"), + )) + .expect("should build auction request"); + + let _ = handle_auction( + &settings, + &orchestrator, + Some(&graph), + Some(®istry), + &mut ec_context, + &noop_services(), + req, + ) + .await; + let mut response = http::Response::new(EdgeBody::empty()); + crate::ec::finalize::ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + Some(&cookie), + None, + &mut response, + ); + + assert_eq!( + stored_ids(&graph, &ec_id), + vec![("body.example.com".to_owned(), "body-uid".to_owned())], + "KV should learn the body's source and not the cookie-only source" + ); + } + + #[tokio::test] + async fn disabled_auction_ingests_nothing_from_the_cookie_but_keeps_shared_id() { + let settings = create_test_settings(); + let orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: false, + ..Default::default() + }); + let registry = ingestion_registry("sharedid.org"); + let ec_id = format!("{}.ingdis", "a".repeat(64)); + let graph = seeded_graph(&ec_id); + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + ec_context.set_eid_sync_source(crate::ec::EidSyncSource::Auction); + let cookie = cookie_only_ts_eids(); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .header(header::COOKIE, format!("{COOKIE_TS_EIDS}={cookie}")) + .body(EdgeBody::from(ad_units_only_body())) + .expect("should build auction request"); + + let _ = handle_auction( + &settings, + &orchestrator, + Some(&graph), + Some(®istry), + &mut ec_context, + &noop_services(), + req, + ) + .await; + let mut response = http::Response::new(EdgeBody::empty()); + crate::ec::finalize::ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + Some(&cookie), + Some("shared-cookie-id"), + &mut response, + ); + + assert_eq!( + stored_ids(&graph, &ec_id), + vec![("sharedid.org".to_owned(), "shared-cookie-id".to_owned())], + "an /auction that resolved no client set should ingest only the sharedId cookie" + ); + } + + #[test] + fn navigation_ingests_the_cookie_only_source() { + // Control for the two tests above: the registry does match the + // cookie-only source when the route reads the cookie. + let settings = create_test_settings(); + let registry = ingestion_registry("body.example.com"); + let ec_id = format!("{}.ingnav", "a".repeat(64)); + let graph = seeded_graph(&ec_id); + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + ec_context.set_eid_sync_source(crate::ec::EidSyncSource::Navigation); + let cookie = cookie_only_ts_eids(); + let mut response = http::Response::new(EdgeBody::empty()); + + crate::ec::finalize::ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + Some(&cookie), + None, + &mut response, + ); + + assert_eq!( + stored_ids(&graph, &ec_id), + vec![(COOKIE_ONLY_SOURCE.to_owned(), "cookie-uid".to_owned())], + "a navigation should ingest the cookie's registered source" + ); + } + #[tokio::test] async fn auction_body_eids_reach_kv_even_when_the_ts_eids_cookie_is_absent() { // Regression test for #1184: `/auction` sends every EID it has in the @@ -1295,6 +1386,197 @@ mod tests { } } + const AUCTION_BODY_FIXTURE: &str = + include_str!("../../../trusted-server-js/lib/test/fixtures/auction-body-eids.txt"); + const TS_EIDS_COOKIE_FIXTURE: &str = + include_str!("../../../trusted-server-js/lib/test/fixtures/ts-eids-cookie.txt"); + + /// Provider that records the full EID set its auction request carried, + /// then fails its launch so no real transport handle is needed. + struct EidRecordingProvider { + eids: Arc>>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for EidRecordingProvider { + fn provider_name(&self) -> &str { + "eid_recording_provider" + } + + async fn request_bids( + &self, + request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + *self.eids.lock().expect("should lock recorded eids") = Some( + serde_json::to_value(&request.user.eids).expect("should serialize recorded eids"), + ); + Err(Report::new(TrustedServerError::Auction { + message: "record only".to_string(), + })) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("parse_response must not run when the launch fails"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("record-backend".to_string()) + } + } + + /// Runs `/auction` with an allowed EC and returns the EIDs the provider + /// received: `None` when no provider was called (the request was + /// rejected), `Some(Value::Null)` when it carried no EIDs. + async fn recorded_auction_eids( + body: Vec, + ts_eids_cookie: Option<&str>, + ) -> Option { + let settings = create_test_settings(); + let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["eid_recording_provider"]), + timeout_ms: 2000, + mediator: None, + ..Default::default() + }); + let recorded = Arc::new(Mutex::new(None)); + orchestrator.register_provider(Arc::new(EidRecordingProvider { + eids: Arc::clone(&recorded), + })); + let ec_id = format!("{}.fixtur", "a".repeat(64)); + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let mut builder = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction"); + if let Some(cookie) = ts_eids_cookie { + builder = builder.header(header::COOKIE, format!("{COOKIE_TS_EIDS}={cookie}")); + } + let req = builder + .body(EdgeBody::from(body)) + .expect("should build auction request"); + + // The recording provider fails its launch; the assertion is on what + // it received, not on the auction result. + let _ = handle_auction( + &settings, + &orchestrator, + None, + None, + &mut ec_context, + &noop_services(), + req, + ) + .await; + + recorded.lock().expect("should lock recorded eids").clone() + } + + /// The two parsable entries of the fixture input, as providers receive + /// them. The cookie copy has no EID-level `ext` on the last entry. + fn fixture_good_entries(with_eid_ext: bool) -> JsonValue { + let mut last = json!({ + "source": "last.example.com", + "uids": [{ "id": "last-uid", "atype": 3 }] + }); + if with_eid_ext { + last["ext"] = json!({ "segments": ["s1"] }); + } + json!([ + { + "source": "first.example.com", + "inserter": "inserter-€.example.com", + "matcher": "first.example.com", + "mm": 3, + "uids": [{ "id": "first-uid", "atype": 1, "ext": { "label": "日本" } }] + }, + last + ]) + } + + #[tokio::test] + async fn tsjs_auction_body_fixture_drops_only_the_unparsable_entry() { + let recorded = + recorded_auction_eids(AUCTION_BODY_FIXTURE.trim_end().as_bytes().to_vec(), None).await; + + assert_eq!( + recorded, + Some(fixture_good_entries(true)), + "the request should be accepted and carry the two good entries unchanged" + ); + } + + #[tokio::test] + async fn tsjs_cookie_fixture_drops_only_the_unparsable_entry() { + let recorded = recorded_auction_eids( + ad_units_only_body(), + Some(TS_EIDS_COOKIE_FIXTURE.trim_end()), + ) + .await; + + assert_eq!( + recorded, + Some(fixture_good_entries(false)), + "the cookie's good entries should reach the provider without the bad one" + ); + } + + #[tokio::test] + async fn auction_body_eid_level_fields_reach_the_auction_request() { + let eids = json!([{ + "source": "adserver.org", + "inserter": "adserver.org", + "matcher": "adserver.org", + "mm": 4, + "uids": [{ "id": "fictional-tdid", "atype": 1, "ext": { "rtiPartner": "TDID" } }] + }]); + let body = serde_json::to_vec(&json!({ + "adUnits": [{ "code": "div-gpt-1", "mediaTypes": { "banner": { "sizes": [[300, 250]] } } }], + "eids": eids + })) + .expect("should serialize body"); + + assert_eq!( + recorded_auction_eids(body, None).await, + Some(eids), + "EID-level fields from the body should reach the provider unchanged" + ); + } + + fn ad_units_only_body() -> Vec { + serde_json::to_vec(&json!({ + "adUnits": [{ "code": "div-gpt-1", "mediaTypes": { "banner": { "sizes": [[300, 250]] } } }] + })) + .expect("should serialize body") + } + + #[tokio::test] + async fn auction_uses_the_cookie_when_no_body_entry_is_valid() { + let cookie_entries = + json!([{ "source": "cookie.example.com", "uids": [{ "id": "cookie-uid" }] }]); + let cookie = + BASE64.encode(serde_json::to_vec(&cookie_entries).expect("should serialize cookie")); + let body = serde_json::to_vec(&json!({ + "adUnits": [{ "code": "div-gpt-1", "mediaTypes": { "banner": { "sizes": [[300, 250]] } } }], + "eids": [{ "source": "", "uids": [{ "id": "body-uid" }] }] + })) + .expect("should serialize body"); + + assert_eq!( + recorded_auction_eids(body, Some(&cookie)).await, + Some(cookie_entries), + "a body whose entries are all invalid should count as absent" + ); + } + #[tokio::test] async fn auction_strips_client_eids_when_ec_identity_denied() { // US-state opt-out via GPC: the server-side auction consent gate still @@ -1482,12 +1764,13 @@ mod tests { #[test] fn resolve_client_auction_eids_prefers_request_body_over_cookie() { - let raw = json!([ + let raw = serde_json::value::to_raw_value(&json!([ { "source": "id5-sync.com", "uids": [{ "id": "body_uid", "atype": 1 }] } - ]); + ])) + .expect("should capture raw EIDs"); let cookie_payload = json!([ { "source": "sharedid.org", @@ -1497,258 +1780,280 @@ mod tests { let encoded = BASE64 .encode(serde_json::to_vec(&cookie_payload).expect("should serialize cookie payload")); - let resolved = resolve_client_auction_eids(Some(&raw), Some(&encoded)) + let resolved = resolve_client_auction_eids(Some(raw.as_ref()), Some(&encoded)) .expect("should prefer request body EIDs"); - assert_eq!(resolved.len(), 1, "should use request body when present"); - assert_eq!(resolved[0].source, "id5-sync.com"); - assert_eq!(resolved[0].uids[0].id, "body_uid"); + assert_eq!( + serde_json::to_value(&resolved).expect("should serialize resolved EIDs"), + json!([{ "source": "id5-sync.com", "uids": [{ "id": "body_uid", "atype": 1 }] }]), + "should use the request body when it has valid entries" + ); } - #[test] - fn parse_client_auction_eids_ignores_malformed_entries() { - let raw = json!([ - { - "source": "id5-sync.com", - "uids": [{ "id": "ID5_abc", "atype": 1 }] - }, - { - "source": "broken.example", - "uids": "not-an-array" - }, - { - "source": "sharedid.org", - "uids": [{ "id": "shared_123" }, { "id": "" }] - } - ]); + const SOURCE_S: &str = "ids.example.com"; + const SOURCE_T: &str = "other.example.com"; - let parsed = parse_client_auction_eids(Some(&raw)).expect("should parse valid EIDs"); + fn kv_eid(source: &str, id: &str, atype: i32) -> Eid { + Eid::new( + source.to_owned(), + vec![Uid { + id: id.to_owned(), + atype: Some(atype), + ext: None, + }], + ) + } - assert_eq!(parsed.len(), 2, "should keep only valid EID entries"); - assert_eq!(parsed[0].source, "id5-sync.com"); - assert_eq!(parsed[0].uids.len(), 1, "should keep valid UID"); - assert_eq!(parsed[1].source, "sharedid.org"); - assert_eq!(parsed[1].uids.len(), 1, "should drop empty UID values"); + /// A browser entry, every field spelled out (spec 7.1 rule 9). + fn browser_eid( + source: &str, + id: &str, + atype: Option, + inserter: Option<&str>, + mm: Option, + ) -> Eid { + Eid { + source: source.to_owned(), + uids: vec![Uid { + id: id.to_owned(), + atype, + ext: None, + }], + inserter: inserter.map(str::to_owned), + matcher: None, + mm, + ext: None, + } } - #[test] - fn parse_client_auction_eids_caps_sources_and_uids() { - let entries: Vec<_> = (0..(MAX_CLIENT_EID_SOURCES + 5)) - .map(|source_index| { - let uids: Vec<_> = (0..(MAX_CLIENT_UIDS_PER_SOURCE + 5)) - .map(|uid_index| json!({ "id": format!("uid-{source_index}-{uid_index}") })) - .collect(); - json!({ - "source": format!("source-{source_index}.example.com"), - "uids": uids, - }) - }) - .collect(); - let raw = JsonValue::Array(entries); + fn merged_json(client: Option>, kv: Option>) -> JsonValue { + serde_json::to_value(merge_auction_eids(client, kv)).expect("should serialize merged EIDs") + } - let parsed = parse_client_auction_eids(Some(&raw)).expect("should parse capped EIDs"); + #[test] + fn merge_case_1_forwards_a_browser_entry_with_its_fields() { + let merged = merged_json( + Some(vec![browser_eid( + SOURCE_S, + "u1", + None, + Some("inserter-1.example.com"), + Some(1), + )]), + None, + ); assert_eq!( - parsed.len(), - MAX_CLIENT_EID_SOURCES, - "should cap client EID sources" - ); - assert!( - parsed - .iter() - .all(|eid| eid.uids.len() == MAX_CLIENT_UIDS_PER_SOURCE), - "should cap UIDs per source" + merged, + json!([{ "source": SOURCE_S, "inserter": "inserter-1.example.com", "mm": 1, "uids": [{ "id": "u1" }] }]), + "a browser entry should go out with its EID-level fields" ); } #[test] - fn parse_client_auction_eids_drops_whitespace_and_oversized_uids() { - let raw = json!([ - { - "source": "id5-sync.com", - "uids": [ - { "id": " " }, - { "id": "x".repeat(MAX_UID_LENGTH + 1) }, - { "id": "valid" } - ] - } - ]); - - let parsed = parse_client_auction_eids(Some(&raw)).expect("should parse valid UID"); + fn merge_case_2_keeps_the_browser_atype_for_a_shared_uid() { + let merged = merged_json( + Some(vec![browser_eid( + SOURCE_S, + "u1", + Some(1), + Some("inserter-1.example.com"), + Some(1), + )]), + Some(vec![kv_eid(SOURCE_S, "u1", 3)]), + ); - assert_eq!(parsed.len(), 1, "should retain source with valid UID"); - assert_eq!(parsed[0].uids.len(), 1, "should drop invalid UIDs"); - assert_eq!(parsed[0].uids[0].id, "valid", "should keep valid UID"); + assert_eq!( + merged, + json!([{ "source": SOURCE_S, "inserter": "inserter-1.example.com", "mm": 1, "uids": [{ "id": "u1", "atype": 1 }] }]), + "KV's operator-configured atype should not override the browser's" + ); } #[test] - fn parse_client_auction_eids_preserves_pair_atype() { - let raw = json!([ - { - "source": "google.com", - "uids": [{ "id": "pair-id", "atype": 571187 }] - } - ]); - - let parsed = parse_client_auction_eids(Some(&raw)).expect("should parse PAIR EID"); + fn merge_case_3_drops_the_stale_kv_uid_for_a_browser_source() { + let merged = merged_json( + Some(vec![browser_eid( + SOURCE_S, + "u1", + None, + Some("inserter-1.example.com"), + Some(1), + )]), + Some(vec![kv_eid(SOURCE_S, "u0", 3)]), + ); assert_eq!( - parsed[0].uids[0].atype, - Some(571187), - "should preserve PAIR's vendor-specific atype" + merged, + json!([{ "source": SOURCE_S, "inserter": "inserter-1.example.com", "mm": 1, "uids": [{ "id": "u1" }] }]), + "KV's value for a source the browser sends should not be forwarded" ); } #[test] - fn parse_client_auction_eids_preserves_uid_ext_and_sanitizes_invalid_atype() { - let raw = json!([ - { - "source": "adserver.org", - "uids": [ - { - "id": "uid-with-ext", - "atype": 1, - "ext": { "provider": "liveintent.com", "rtiPartner": "TDID" } - }, - { - "id": "uid-bad-atype", - "atype": 2_147_483_648_u64, - "ext": { "keep": true } - }, - { - "id": "uid-float-atype", - "atype": 1.5 - } - ] - } - ]); - - let parsed = parse_client_auction_eids(Some(&raw)).expect("should parse valid EIDs"); + fn merge_case_4_appends_kv_sources_the_browser_did_not_send() { + let merged = merged_json( + Some(vec![browser_eid(SOURCE_S, "u1", None, None, None)]), + Some(vec![kv_eid(SOURCE_T, "t1", 3)]), + ); - assert_eq!(parsed.len(), 1, "should keep valid source"); - assert_eq!(parsed[0].uids.len(), 3, "should keep valid UIDs"); assert_eq!( - parsed[0].uids[0].atype, - Some(1), - "should preserve valid atype" + merged, + json!([ + { "source": SOURCE_S, "uids": [{ "id": "u1" }] }, + { "source": SOURCE_T, "uids": [{ "id": "t1", "atype": 3 }] } + ]), + "browser entries should come first and KV-only sources after" ); + } + + #[test] + fn merge_case_5_keeps_same_source_entries_with_different_fields_apart() { + let merged = merged_json( + Some(vec![ + browser_eid( + SOURCE_S, + "u1", + None, + Some("inserter-1.example.com"), + Some(1), + ), + browser_eid( + SOURCE_S, + "u1", + None, + Some("inserter-2.example.com"), + Some(2), + ), + ]), + None, + ); + assert_eq!( - parsed[0].uids[0].ext, - Some(json!({ "provider": "liveintent.com", "rtiPartner": "TDID" })), - "should preserve uid ext" + merged, + json!([ + { "source": SOURCE_S, "inserter": "inserter-1.example.com", "mm": 1, "uids": [{ "id": "u1" }] }, + { "source": SOURCE_S, "inserter": "inserter-2.example.com", "mm": 2, "uids": [{ "id": "u1" }] } + ]), + "two browser entries for one source should stay two entries" ); + } + + #[test] + fn merge_case_6_matches_sources_as_the_registry_normalizes_them() { + let merged = merged_json( + Some(vec![browser_eid( + "IDS.Example.com.", + "u1", + None, + None, + None, + )]), + Some(vec![kv_eid(SOURCE_S, "u0", 3)]), + ); + assert_eq!( - parsed[0].uids[1].atype, None, - "should drop out-of-range atype without dropping uid" + merged, + json!([{ "source": "IDS.Example.com.", "uids": [{ "id": "u1" }] }]), + "a browser source differing only in case or a trailing dot should suppress KV's" + ); + } + + #[test] + fn merge_case_7_forwards_kv_entries_when_the_browser_sent_none() { + let merged = merged_json( + None, + Some(vec![kv_eid(SOURCE_S, "u0", 3), kv_eid(SOURCE_T, "t1", 3)]), ); + assert_eq!( - parsed[0].uids[1].ext, - Some(json!({ "keep": true })), - "should preserve ext when atype is invalid" + merged, + json!([ + { "source": SOURCE_S, "uids": [{ "id": "u0", "atype": 3 }] }, + { "source": SOURCE_T, "uids": [{ "id": "t1", "atype": 3 }] } + ]), + "KV entries should be forwarded unchanged without browser EIDs" ); + } + + #[test] + fn merge_drops_entries_without_uids() { + let empty_browser = Eid::new(SOURCE_S.to_owned(), Vec::new()); + let empty_kv = Eid::new(SOURCE_T.to_owned(), Vec::new()); + assert_eq!( - parsed[0].uids[2].atype, None, - "should drop non-integer atype without dropping uid" + merged_json(Some(vec![empty_browser]), Some(vec![empty_kv])), + JsonValue::Null, + "entries without UIDs should be dropped, leaving no EIDs" ); } #[test] fn merge_auction_eids_deduplicates_client_and_resolved_ids() { - let client_eids = Some(vec![Eid { - source: "id5-sync.com".to_string(), - uids: vec![Uid { - id: "ID5_abc".to_string(), - atype: Some(1), - ext: None, - }], - }]); - let resolved_eids = Some(vec![ - Eid { - source: "id5-sync.com".to_string(), - uids: vec![Uid { - id: "ID5_abc".to_string(), - atype: Some(1), - ext: None, - }], - }, - Eid { - source: "liveramp.com".to_string(), - uids: vec![Uid { - id: "LR_xyz".to_string(), - atype: Some(3), - ext: None, - }], - }, - ]); - - let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); + let merged = merged_json( + Some(vec![kv_eid("id5-sync.com", "ID5_abc", 1)]), + Some(vec![ + kv_eid("id5-sync.com", "ID5_abc", 1), + kv_eid("liveramp.com", "LR_xyz", 3), + ]), + ); - assert_eq!(merged.len(), 2, "should retain distinct EID sources"); - assert_eq!(merged[0].source, "id5-sync.com"); - assert_eq!(merged[0].uids.len(), 1, "should deduplicate matching UIDs"); - assert_eq!(merged[1].source, "liveramp.com"); - assert_eq!(merged[1].uids[0].id, "LR_xyz"); + assert_eq!( + merged, + json!([ + { "source": "id5-sync.com", "uids": [{ "id": "ID5_abc", "atype": 1 }] }, + { "source": "liveramp.com", "uids": [{ "id": "LR_xyz", "atype": 3 }] } + ]), + "the browser's id5 entry should go out once, followed by KV's liveramp" + ); } #[test] - fn merge_auction_eids_preserves_multiple_uids_per_source() { - let client_eids = Some(vec![Eid { - source: "sharedid.org".to_string(), - uids: vec![Uid { - id: "shared_client".to_string(), - atype: None, - ext: None, - }], - }]); - let resolved_eids = Some(vec![Eid { - source: "sharedid.org".to_string(), - uids: vec![Uid { - id: "shared_server".to_string(), - atype: Some(3), - ext: None, - }], - }]); - - let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); + fn merge_auction_eids_does_not_union_kv_uids_into_a_browser_source() { + let merged = merged_json( + Some(vec![browser_eid( + "sharedid.org", + "shared_client", + None, + None, + None, + )]), + Some(vec![kv_eid("sharedid.org", "shared_server", 3)]), + ); - assert_eq!(merged.len(), 1, "should merge same-source entries"); - assert_eq!(merged[0].uids.len(), 2, "should preserve distinct UIDs"); - assert_eq!(merged[0].uids[0].id, "shared_server"); - assert_eq!(merged[0].uids[1].id, "shared_client"); + assert_eq!( + merged, + json!([{ "source": "sharedid.org", "uids": [{ "id": "shared_client" }] }]), + "KV's UID for a source the browser sends should not be added to it" + ); } #[test] - fn merge_auction_eids_prefers_server_resolved_metadata_on_conflict() { - let client_eids = Some(vec![Eid { - source: "adserver.org".to_string(), - uids: vec![Uid { - id: "shared_uid".to_string(), + fn merge_auction_eids_keeps_browser_metadata_on_conflict() { + let client = Eid::new( + "adserver.org".to_owned(), + vec![Uid { + id: "shared_uid".to_owned(), atype: Some(1), ext: Some(json!({ "provider": "client" })), }], - }]); - let resolved_eids = Some(vec![Eid { - source: "adserver.org".to_string(), - uids: vec![Uid { - id: "shared_uid".to_string(), + ); + let resolved = Eid::new( + "adserver.org".to_owned(), + vec![Uid { + id: "shared_uid".to_owned(), atype: Some(3), ext: Some(json!({ "provider": "server" })), }], - }]); - - let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); - - assert_eq!(merged.len(), 1, "should merge duplicate source"); - assert_eq!(merged[0].uids.len(), 1, "should deduplicate duplicate uid"); - assert_eq!( - merged[0].uids[0].atype, - Some(3), - "should prefer resolved atype" ); + assert_eq!( - merged[0].uids[0].ext, - Some(json!({ "provider": "server" })), - "should prefer resolved ext" + merged_json(Some(vec![client]), Some(vec![resolved])), + json!([{ + "source": "adserver.org", + "uids": [{ "id": "shared_uid", "atype": 1, "ext": { "provider": "client" } }] + }]), + "the browser's atype and ext should win over KV's" ); } diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index a0df63d94a..cec4823005 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -9,6 +9,7 @@ use error_stack::{Report, ResultExt, ensure}; use http::{HeaderValue, Request, Response, StatusCode, header}; use serde::Deserialize; use serde_json::Value as JsonValue; +use serde_json::value::RawValue; use std::collections::{BTreeMap, HashMap, HashSet}; use url::Url; use uuid::Uuid; @@ -43,7 +44,10 @@ use super::types::{ pub struct AdRequest { pub ad_units: Vec, pub config: Option, - pub eids: Option, + /// Browser EIDs, captured as raw JSON so a malformed entry (for example a + /// lone surrogate escape) cannot fail the whole body. Each entry is + /// validated on its own by the shared client EID parser. + pub eids: Option>, } /// A single ad placement in an [`AdRequest`]. @@ -558,12 +562,16 @@ mod tests { AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, }; use crate::auction::routing::route_auction; + use crate::auction::test_support::{ + repeated_source_provenance_eids, repeated_source_provenance_eids_json, + }; use crate::auction::types::{ ApsRendererV1, ApsTagType, AuctionResponse, Bid, BidRenderer, BidStatus, }; use crate::openrtb::{Eid, Uid}; use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; + use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use http::Method; use serde_json::json; use std::collections::{BTreeMap, BTreeSet, HashSet}; @@ -810,17 +818,44 @@ mod tests { ); } + #[test] + fn eid_header_carries_provenance_fields_and_repeated_sources() { + let mut request = make_auction_request(); + request.user.eids = Some(repeated_source_provenance_eids()); + + let response = + convert_to_openrtb_response(&make_empty_result(), &make_settings(), &request, true) + .expect("should build response"); + + let header = response + .headers() + .get(&HEADER_X_TS_EIDS) + .and_then(|value| value.to_str().ok()) + .expect("should include x-ts-eids"); + let decoded: serde_json::Value = serde_json::from_slice( + &BASE64 + .decode(header) + .expect("should decode x-ts-eids base64"), + ) + .expect("should parse x-ts-eids JSON"); + assert_eq!( + decoded, + repeated_source_provenance_eids_json(), + "x-ts-eids should show both entries with their own EID-level fields" + ); + } + #[test] fn response_includes_eid_headers_when_eids_present() { let mut request = make_auction_request(); - request.user.eids = Some(vec![Eid { - source: "ssp.com".to_owned(), - uids: vec![Uid { + request.user.eids = Some(vec![Eid::new( + "ssp.com".to_owned(), + vec![Uid { id: "uid-1".to_owned(), atype: Some(3), ext: None, }], - }]); + )]); let settings = make_settings(); let result = make_empty_result(); diff --git a/crates/trusted-server-core/src/auction/openrtb/tests.rs b/crates/trusted-server-core/src/auction/openrtb/tests.rs index f47efa475e..1ed5683169 100644 --- a/crates/trusted-server-core/src/auction/openrtb/tests.rs +++ b/crates/trusted-server-core/src/auction/openrtb/tests.rs @@ -1569,3 +1569,18 @@ fn malformed_top_level_standard_response_is_error() { assert_eq!(response.status, BidStatus::Error); assert_eq!(response.metadata["error_type"], "parse_response"); } + +#[test] +fn standard_profile_forwards_eid_provenance_fields_and_repeated_sources() { + let mut request = canonical_parity_auction_request(); + request.user.eids = Some(crate::auction::test_support::repeated_source_provenance_eids()); + + let built = build_with_request("standard", json!({}), request, None); + + let serialized = serde_json::to_value(&built).expect("should serialize request"); + assert_eq!( + serialized["user"]["ext"]["eids"], + crate::auction::test_support::repeated_source_provenance_eids_json(), + "config-first providers should receive both entries with their own EID-level fields" + ); +} diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index a83f8899a8..5d44d906a1 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -92,14 +92,14 @@ pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { raw_ac_string: Some("fictional-ac".to_string()), ..Default::default() }), - eids: Some(vec![Eid { - source: "identity.example".to_string(), - uids: vec![Uid { + eids: Some(vec![Eid::new( + "identity.example".to_string(), + vec![Uid { id: "fictional-uid".to_string(), atype: Some(1), ext: None, }], - }]), + )]), }, device: Some(DeviceInfo { user_agent: Some("Fictional Browser".to_string()), @@ -119,3 +119,56 @@ pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { context: HashMap::new(), } } + +/// Two entries for one source with different EID-level fields, the way +/// Prebid sends them (spec 3.2), for outbound serialization tests. +pub(crate) fn repeated_source_provenance_eids() -> Vec { + vec![ + Eid { + source: "ids.example.com".to_string(), + uids: vec![Uid { + id: "fictional-uid".to_string(), + atype: Some(1), + ext: None, + }], + inserter: Some("inserter-a.example.com".to_string()), + matcher: Some("ids.example.com".to_string()), + mm: Some(4), + ext: None, + }, + Eid { + source: "ids.example.com".to_string(), + uids: vec![Uid { + id: "fictional-uid".to_string(), + atype: Some(1), + ext: None, + }], + inserter: Some("inserter-b.example.com".to_string()), + matcher: Some("matcher-b.example.com".to_string()), + mm: Some(2), + ext: Some(json!({ "segments": ["s1"] })), + }, + ] +} + +/// [`repeated_source_provenance_eids`] as the JSON every outbound path must +/// emit in `user.ext.eids`. +pub(crate) fn repeated_source_provenance_eids_json() -> serde_json::Value { + json!([ + { + "source": "ids.example.com", + "uids": [{ "id": "fictional-uid", "atype": 1 }], + "inserter": "inserter-a.example.com", + "matcher": "ids.example.com", + "mm": 4 + }, + { + "source": "ids.example.com", + "uids": [{ "id": "fictional-uid", "atype": 1 }], + "inserter": "inserter-b.example.com", + "matcher": "matcher-b.example.com", + "mm": 2, + "ext": { "segments": ["s1"] } + } + ]) +} diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index e21fab702c..18b5563307 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -1478,6 +1478,33 @@ mod tests { ); } + #[test] + fn eids_lookup_reports_non_array_cookie_as_parse_error() { + let payload = BASE64.encode(br#"{"a":1}"#); + let req = get_request_with_cookie("/_ts/admin/eids", &format!("ts-eids={payload}")); + + let response = + handle_admin_eids_lookup(&test_registry(), &req).expect("should handle eids lookup"); + + let json = response_json(response); + assert_eq!(json["cookie_present"], true); + assert!( + json["parse_error"] + .as_str() + .expect("should have parse_error") + .contains("ts-eids"), + "should describe the parse failure" + ); + assert!(json.get("eids").is_none(), "should omit unparsed eids"); + assert!( + json["ingest"]["matched"] + .as_array() + .expect("should have matched list") + .is_empty(), + "non-array cookie should preview no matches" + ); + } + #[test] fn eids_lookup_reports_parse_error() { let req = get_request_with_cookie("/_ts/admin/eids", "ts-eids=!!!not-base64!!!"); diff --git a/crates/trusted-server-core/src/ec/client_eids.rs b/crates/trusted-server-core/src/ec/client_eids.rs new file mode 100644 index 0000000000..30951b9d21 --- /dev/null +++ b/crates/trusted-server-core/src/ec/client_eids.rs @@ -0,0 +1,600 @@ +//! Parsing and validation of browser-supplied EIDs. +//! +//! The `/auction` request body and the `ts-eids` cookie carry the same +//! OpenRTB-style EID array, which TSJS copies from the EIDs Prebid placed in +//! the `trustedServer` bidder request. Both go through [`parse_client_eids`], +//! so a malformed entry or field costs only itself, whichever transport or +//! client sent it. +//! +//! Rules, per entry: +//! +//! - The entry parses on its own and is a JSON object, or it is dropped. +//! - `source` is a string, not blank, at most [`MAX_CLIENT_EID_LABEL_BYTES`] +//! UTF-8 bytes, or the entry is dropped. +//! - `uids` holds at least one valid UID, or the entry is dropped. The first +//! [`MAX_CLIENT_UIDS_PER_ENTRY`] valid UIDs are kept. +//! - `uids[].id` is a string, not blank, at most `MAX_UID_LENGTH` bytes, or +//! the UID is dropped. `uids[].atype` is an integer from 0 to `i32::MAX`; +//! `uids[].ext` is an object of at most [`MAX_CLIENT_EID_EXT_BYTES`] as +//! compact JSON. An invalid `atype` or `ext` drops only that field. +//! - `inserter` and `matcher` follow the `source` rule, `mm` the `atype` rule +//! and `ext` the UID `ext` rule. An invalid value drops only that field. +//! - Unknown keys are ignored. Order is preserved and a source may repeat. +//! +//! An entry equal to an earlier kept entry is dropped, and then the first +//! [`MAX_CLIENT_EID_ENTRIES`] entries are kept. "Blank" means empty or only +//! whitespace by `str::trim`. "Integer" means a JSON number with no fractional +//! part, so `4`, `4.0` and `4e0` are the same value. + +use serde_json::Value as JsonValue; +use serde_json::value::RawValue; + +use crate::openrtb::{Eid, Uid}; + +use super::kv_types::MAX_UID_LENGTH; + +/// Maximum number of entries kept from one browser EID array. +pub(crate) const MAX_CLIENT_EID_ENTRIES: usize = 64; + +/// Maximum number of valid UIDs kept per entry. +pub(crate) const MAX_CLIENT_UIDS_PER_ENTRY: usize = 32; + +/// Maximum UTF-8 length of `source`, `inserter` and `matcher`. +pub(crate) const MAX_CLIENT_EID_LABEL_BYTES: usize = 255; + +/// Maximum compact-JSON length of an EID-level or UID-level `ext` object. +pub(crate) const MAX_CLIENT_EID_EXT_BYTES: usize = 2048; + +/// Parses and validates browser-supplied EID entries from an `/auction` body +/// or a `ts-eids` cookie. +/// +/// See the module documentation for the rules. Anything other than a JSON +/// array yields no entries. +/// +/// # Performance +/// +/// Parses each element into a [`JsonValue`] once. Duplicate detection is +/// quadratic in the kept entries, which are capped at +/// [`MAX_CLIENT_EID_ENTRIES`]. +#[must_use] +pub(crate) fn parse_client_eids(raw: &RawValue) -> Vec { + // Capturing elements as raw JSON does not validate string escapes, so an + // element holding a lone surrogate fails only its own parse below. + let Ok(elements) = serde_json::from_str::>>(raw.get()) else { + log::debug!("Client EIDs: payload is not a JSON array; ignoring it"); + return Vec::new(); + }; + + let mut eids: Vec = Vec::new(); + for element in &elements { + if eids.len() >= MAX_CLIENT_EID_ENTRIES { + log::debug!( + "Client EIDs: keeping the first {MAX_CLIENT_EID_ENTRIES} entries; dropping the rest" + ); + break; + } + let Some(eid) = parse_client_eid(element) else { + continue; + }; + if eids.contains(&eid) { + log::debug!( + "Client EIDs: dropping a duplicate entry for source '{}'", + eid.source + ); + continue; + } + eids.push(eid); + } + eids +} + +fn parse_client_eid(element: &RawValue) -> Option { + let entry = match serde_json::from_str::(element.get()) { + Ok(JsonValue::Object(entry)) => entry, + Ok(_) => { + log::debug!("Client EIDs: dropping an entry that is not a JSON object"); + return None; + } + Err(_) => { + log::debug!("Client EIDs: dropping an entry that does not parse"); + return None; + } + }; + + let Some(source) = entry.get("source").and_then(parse_client_eid_label) else { + log::debug!("Client EIDs: dropping an entry without a valid `source`"); + return None; + }; + + let uids: Vec = match entry.get("uids") { + Some(JsonValue::Array(raw_uids)) => raw_uids + .iter() + .filter_map(|raw_uid| parse_client_uid(raw_uid, &source)) + .take(MAX_CLIENT_UIDS_PER_ENTRY) + .collect(), + _ => Vec::new(), + }; + if uids.is_empty() { + log::debug!("Client EIDs: dropping source '{source}' without a valid UID"); + return None; + } + + let inserter = parse_optional_field(&entry, "EID", "inserter", &source, parse_client_eid_label); + let matcher = parse_optional_field(&entry, "EID", "matcher", &source, parse_client_eid_label); + let mm = parse_optional_field(&entry, "EID", "mm", &source, parse_client_eid_integer); + let ext = parse_optional_field(&entry, "EID", "ext", &source, parse_client_eid_ext); + + Some(Eid { + source, + uids, + inserter, + matcher, + mm, + ext, + }) +} + +fn parse_client_uid(raw: &JsonValue, source: &str) -> Option { + let JsonValue::Object(uid) = raw else { + log::debug!("Client EIDs: dropping a UID that is not an object for source '{source}'"); + return None; + }; + + let Some(id) = uid + .get("id") + .and_then(JsonValue::as_str) + .filter(|id| !id.trim().is_empty()) + .filter(|id| id.len() <= MAX_UID_LENGTH) + else { + log::debug!("Client EIDs: dropping a UID without a valid `id` for source '{source}'"); + return None; + }; + + Some(Uid { + id: id.to_owned(), + atype: parse_optional_field(uid, "UID", "atype", source, parse_client_eid_integer), + ext: parse_optional_field(uid, "UID", "ext", source, parse_client_eid_ext), + }) +} + +/// Applies `parse` to `key` when it is present, logging the field name and +/// source, never the value, when the value is invalid. +fn parse_optional_field( + object: &serde_json::Map, + scope: &str, + key: &str, + source: &str, + parse: impl Fn(&JsonValue) -> Option, +) -> Option { + let value = object.get(key)?; + let parsed = parse(value); + if parsed.is_none() { + log::debug!("Client EIDs: dropping invalid {scope} `{key}` for source '{source}'"); + } + parsed +} + +/// Reads `source`, `inserter` or `matcher`: a string that is not blank and at +/// most [`MAX_CLIENT_EID_LABEL_BYTES`] long, kept as supplied. +fn parse_client_eid_label(value: &JsonValue) -> Option { + value + .as_str() + .filter(|label| !label.trim().is_empty()) + .filter(|label| label.len() <= MAX_CLIENT_EID_LABEL_BYTES) + .map(str::to_owned) +} + +/// Reads `atype` or `mm`: a JSON number with no fractional part, from 0 to +/// `i32::MAX`. +fn parse_client_eid_integer(value: &JsonValue) -> Option { + let number = value.as_number()?; + if let Some(integer) = number.as_i64() { + return i32::try_from(integer).ok().filter(|integer| *integer >= 0); + } + if number.is_u64() { + // Above `i64::MAX`, so above the accepted range. + return None; + } + // serde_json reads `4.0`, `4e0` and `-0` as floats. + let float = number.as_f64()?; + if float.fract() != 0.0 || !(0.0..=f64::from(i32::MAX)).contains(&float) { + return None; + } + // Integral and within `0..=i32::MAX`, so the conversion is exact. + Some(float as i32) +} + +/// Reads an `ext`: a JSON object of at most [`MAX_CLIENT_EID_EXT_BYTES`] as +/// compact JSON. +fn parse_client_eid_ext(value: &JsonValue) -> Option { + if !value.is_object() { + return None; + } + let compact_len = serde_json::to_vec(value).ok()?.len(); + (compact_len <= MAX_CLIENT_EID_EXT_BYTES).then(|| value.clone()) +} + +#[cfg(test)] +mod tests { + use std::sync::{Mutex, Once}; + + use serde_json::json; + + use super::*; + + fn parse(text: &str) -> Vec { + let raw = RawValue::from_string(text.to_owned()).expect("should capture raw JSON"); + parse_client_eids(&raw) + } + + fn parsed_json(text: &str) -> JsonValue { + serde_json::to_value(parse(text)).expect("should serialize parsed EIDs") + } + + fn good(source: &str) -> String { + format!(r#"{{"source":"{source}","uids":[{{"id":"uid-{source}"}}]}}"#) + } + + fn good_json(source: &str) -> JsonValue { + json!({ "source": source, "uids": [{ "id": format!("uid-{source}") }] }) + } + + /// An `ext` object whose compact JSON is exactly `bytes` long. + fn ext_of_len(bytes: usize) -> String { + // `{"k":""}` is 8 bytes. + format!(r#"{{"k":"{}"}}"#, "x".repeat(bytes - 8)) + } + + static CAPTURED_LOGS: Mutex> = Mutex::new(Vec::new()); + static INSTALL_LOGGER: Once = Once::new(); + + struct CapturingLogger; + + impl log::Log for CapturingLogger { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { + metadata.level() <= log::Level::Debug + } + + fn log(&self, record: &log::Record<'_>) { + if self.enabled(record.metadata()) { + CAPTURED_LOGS + .lock() + .expect("should lock captured logs") + .push(record.args().to_string()); + } + } + + fn flush(&self) {} + } + + fn install_capturing_logger() { + INSTALL_LOGGER.call_once(|| { + log::set_logger(&CapturingLogger).expect("should install the capturing test logger"); + log::set_max_level(log::LevelFilter::Debug); + }); + } + + fn captured_logs() -> Vec { + CAPTURED_LOGS + .lock() + .expect("should lock captured logs") + .clone() + } + + #[test] + fn keeps_a_valid_entry_with_every_field() { + let parsed = parsed_json( + r#"[{"source":"ids.example.com","inserter":"inserter.example.com","matcher":"matcher.example.com","mm":4,"ext":{"segments":["s1"]},"uids":[{"id":"uid-1","atype":1,"ext":{"rtiPartner":"TDID"}}]}]"#, + ); + + assert_eq!( + parsed, + json!([{ + "source": "ids.example.com", + "inserter": "inserter.example.com", + "matcher": "matcher.example.com", + "mm": 4, + "ext": { "segments": ["s1"] }, + "uids": [{ "id": "uid-1", "atype": 1, "ext": { "rtiPartner": "TDID" } }] + }]), + "should keep every valid field" + ); + } + + #[test] + fn yields_nothing_for_a_payload_that_is_not_an_array() { + for text in [ + r#"{"source":"ids.example.com"}"#, + r#""text""#, + "null", + "1", + "true", + ] { + assert_eq!( + parsed_json(text), + json!([]), + "should ignore non-array payload {text}" + ); + } + } + + #[test] + fn drops_only_the_entry_that_does_not_parse() { + // `\ud800` is a lone surrogate escape: serde_json rejects it when the + // element is parsed on its own. + let bad_entries = [ + r#"{"source":"bad-\ud800.example.com","uids":[{"id":"u"}]}"#, + r#"{"source":"bad.example.com","uids":[{"id":"u-\ud800"}]}"#, + r#"{"source":"bad.example.com","inserter":"i-\ud800","uids":[{"id":"u"}]}"#, + r#"{"source":"bad.example.com","matcher":"m-\ud800","uids":[{"id":"u"}]}"#, + r#"{"source":"bad.example.com","ext":{"k-\ud800":1},"uids":[{"id":"u"}]}"#, + r#"{"source":"bad.example.com","ext":{"k":"v-\ud800"},"uids":[{"id":"u"}]}"#, + r#"{"source":"bad.example.com","uids":[{"id":"u","ext":{"k-\ud800":1}}]}"#, + r#"{"source":"bad.example.com","uids":[{"id":"u","ext":{"k":"v-\ud800"}}]}"#, + ]; + for bad in bad_entries { + let text = format!( + "[{},{bad},{}]", + good("a.example.com"), + good("b.example.com") + ); + + assert_eq!( + parsed_json(&text), + json!([good_json("a.example.com"), good_json("b.example.com")]), + "should drop only the unparsable entry {bad}" + ); + } + } + + #[test] + fn drops_entries_that_are_not_objects() { + let text = format!(r#"[1,"text",null,[],true,{}]"#, good("a.example.com")); + + assert_eq!( + parsed_json(&text), + json!([good_json("a.example.com")]), + "should keep only the object entry" + ); + } + + #[test] + fn validates_source() { + let source_255 = format!("a{}", "é".repeat(127)); + let source_256 = "é".repeat(128); + let text = format!( + r#"[{{"uids":[{{"id":"u"}}]}},{{"source":7,"uids":[{{"id":"u"}}]}},{{"source":" ","uids":[{{"id":"u"}}]}},{{"source":"{source_256}","uids":[{{"id":"u"}}]}},{{"source":"{source_255}","uids":[{{"id":"u"}}]}}]"# + ); + + assert_eq!( + parsed_json(&text), + json!([{ "source": source_255, "uids": [{ "id": "u" }] }]), + "should keep only a non-blank string source of at most 255 UTF-8 bytes" + ); + } + + #[test] + fn drops_an_entry_without_a_valid_uid() { + let text = format!( + r#"[{{"source":"no-uids.example.com"}},{{"source":"string-uids.example.com","uids":"u"}},{{"source":"empty-uids.example.com","uids":[]}},{{"source":"bad-uids.example.com","uids":[{{"id":""}},7]}},{}]"#, + good("a.example.com") + ); + + assert_eq!( + parsed_json(&text), + json!([good_json("a.example.com")]), + "should drop entries whose uids hold no valid UID" + ); + } + + #[test] + fn validates_uid_ids() { + let id_512 = "x".repeat(512); + let id_513 = "x".repeat(513); + let text = format!( + r#"[{{"source":"ids.example.com","uids":[{{"id":7}},{{"id":" "}},{{"id":"{id_513}"}},{{"atype":1}},{{"id":"{id_512}"}}]}}]"# + ); + + assert_eq!( + parsed_json(&text), + json!([{ "source": "ids.example.com", "uids": [{ "id": id_512 }] }]), + "should keep only a non-blank string id of at most 512 bytes" + ); + } + + #[test] + fn keeps_the_first_32_valid_uids() { + let uids: Vec = [r#"{"id":""}"#.to_owned(), r#"{"id":7}"#.to_owned()] + .into_iter() + .chain((0..33).map(|index| format!(r#"{{"id":"u{index}"}}"#))) + .collect(); + let text = format!( + r#"[{{"source":"ids.example.com","uids":[{}]}}]"#, + uids.join(",") + ); + + let expected_uids: Vec = (0..32) + .map(|index| json!({ "id": format!("u{index}") })) + .collect(); + assert_eq!( + parsed_json(&text), + json!([{ "source": "ids.example.com", "uids": expected_uids }]), + "should keep the first 32 valid UIDs, not counting invalid ones" + ); + } + + #[test] + fn reads_atype_and_mm_as_integers_in_range() { + let accepted = [ + ("4", 4), + ("4.0", 4), + ("4e0", 4), + ("-0", 0), + ("0", 0), + ("2147483647", 2_147_483_647), + ("2147483647.0", 2_147_483_647), + ("571187", 571_187), + ]; + for (spelling, value) in accepted { + let text = format!( + r#"[{{"source":"ids.example.com","mm":{spelling},"uids":[{{"id":"u","atype":{spelling}}}]}}]"# + ); + + assert_eq!( + parsed_json(&text), + json!([{ "source": "ids.example.com", "mm": value, "uids": [{ "id": "u", "atype": value }] }]), + "should accept {spelling} as {value}" + ); + } + + let rejected = [ + "2147483648", + "9007199254740993", + "18446744073709551616", + "1.5", + "-1", + "1e300", + r#""1""#, + "null", + "true", + ]; + for spelling in rejected { + let text = format!( + r#"[{{"source":"ids.example.com","mm":{spelling},"uids":[{{"id":"u","atype":{spelling}}}]}}]"# + ); + + assert_eq!( + parsed_json(&text), + json!([{ "source": "ids.example.com", "uids": [{ "id": "u" }] }]), + "should drop only the fields for {spelling}" + ); + } + } + + #[test] + fn validates_ext_objects_and_their_size() { + let ext_2048 = ext_of_len(2048); + let ext_2049 = ext_of_len(2049); + let text = format!( + r#"[{{"source":"keep.example.com","ext":{ext_2048},"uids":[{{"id":"u","ext":{ext_2048}}}]}},{{"source":"big.example.com","ext":{ext_2049},"uids":[{{"id":"u","ext":{ext_2049}}}]}},{{"source":"shape.example.com","ext":[1],"uids":[{{"id":"u","ext":"text"}}]}}]"# + ); + let kept_ext: JsonValue = serde_json::from_str(&ext_2048).expect("should parse ext"); + + assert_eq!( + parsed_json(&text), + json!([ + { "source": "keep.example.com", "ext": kept_ext, "uids": [{ "id": "u", "ext": kept_ext }] }, + { "source": "big.example.com", "uids": [{ "id": "u" }] }, + { "source": "shape.example.com", "uids": [{ "id": "u" }] } + ]), + "should keep objects of at most 2048 compact bytes and drop only the bad fields" + ); + } + + #[test] + fn validates_inserter_and_matcher() { + let label_255 = format!("a{}", "é".repeat(127)); + let label_256 = "é".repeat(128); + let text = format!( + r#"[{{"source":"keep.example.com","inserter":"{label_255}","matcher":"{label_255}","uids":[{{"id":"u"}}]}},{{"source":"drop.example.com","inserter":"{label_256}","matcher":" ","uids":[{{"id":"u"}}]}},{{"source":"type.example.com","inserter":7,"matcher":null,"uids":[{{"id":"u"}}]}}]"# + ); + + assert_eq!( + parsed_json(&text), + json!([ + { "source": "keep.example.com", "inserter": label_255, "matcher": label_255, "uids": [{ "id": "u" }] }, + { "source": "drop.example.com", "uids": [{ "id": "u" }] }, + { "source": "type.example.com", "uids": [{ "id": "u" }] } + ]), + "should keep non-blank strings of at most 255 bytes and drop only the bad fields" + ); + } + + #[test] + fn drops_duplicates_before_the_entry_cap() { + let mut entries: Vec = (0..64) + .map(|index| good(&format!("s{index}.example.com"))) + .collect(); + // Equal to entry 0 once validated: unknown keys are ignored. + entries.insert( + 1, + r#"{"source":"s0.example.com","uids":[{"id":"uid-s0.example.com"}],"unknown":true}"# + .to_owned(), + ); + let text = format!("[{}]", entries.join(",")); + + let expected: Vec = (0..64) + .map(|index| good_json(&format!("s{index}.example.com"))) + .collect(); + assert_eq!( + parsed_json(&text), + JsonValue::Array(expected), + "a duplicate should not take one of the 64 places" + ); + } + + #[test] + fn keeps_the_first_64_entries() { + let entries: Vec = (0..70) + .map(|index| good(&format!("s{index}.example.com"))) + .collect(); + let text = format!("[{}]", entries.join(",")); + + let expected: Vec = (0..64) + .map(|index| good_json(&format!("s{index}.example.com"))) + .collect(); + assert_eq!( + parsed_json(&text), + JsonValue::Array(expected), + "should keep the first 64 entries" + ); + } + + #[test] + fn keeps_order_and_repeated_sources_and_ignores_unknown_keys() { + let parsed = parsed_json( + r#"[{"source":"b.example.com","uids":[{"id":"u","unknown":1}],"unknown":2},{"source":"a.example.com","mm":1,"uids":[{"id":"u"}]},{"source":"a.example.com","mm":2,"uids":[{"id":"u"}]}]"#, + ); + + assert_eq!( + parsed, + json!([ + { "source": "b.example.com", "uids": [{ "id": "u" }] }, + { "source": "a.example.com", "mm": 1, "uids": [{ "id": "u" }] }, + { "source": "a.example.com", "mm": 2, "uids": [{ "id": "u" }] } + ]), + "should keep wire order and both same-source entries" + ); + } + + #[test] + fn logs_dropped_entries_and_fields_by_name_and_source_without_values() { + install_capturing_logger(); + + let parsed = parsed_json( + r#"[{"source":"log-a.example.com","mm":-1,"uids":[{"id":"secret-uid-a","atype":"1"}]},{"source":"log-b.example.com","uids":[{"id":""}]}]"#, + ); + + assert_eq!( + parsed, + json!([{ "source": "log-a.example.com", "uids": [{ "id": "secret-uid-a" }] }]), + "should keep the entry and drop only its invalid fields" + ); + let logs = captured_logs(); + for (field, source) in [ + ("`mm`", "log-a.example.com"), + ("`atype`", "log-a.example.com"), + ("without a valid UID", "log-b.example.com"), + ] { + assert!( + logs.iter() + .any(|line| line.contains(field) && line.contains(source)), + "should log {field} for {source}; captured: {logs:?}" + ); + } + assert!( + !logs.iter().any(|line| line.contains("secret-uid-a")), + "should never log a UID value" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 1359f75091..0493f5dec1 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -64,13 +64,15 @@ pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec Vec { resolved .iter() - .map(|item| Eid { - source: item.source_domain.clone(), - uids: vec![Uid { - id: item.uid.clone(), - atype: Some(item.openrtb_atype), - ext: None, - }], + .map(|item| { + Eid::new( + item.source_domain.clone(), + vec![Uid { + id: item.uid.clone(), + atype: Some(item.openrtb_atype), + ext: None, + }], + ) }) .collect() } diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 019ef2aa40..e41a8180d4 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -10,6 +10,7 @@ use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; use crate::consent::{ConsentContext, allows_eid_persistence}; +use crate::openrtb::Eid; use crate::settings::Settings; use super::EcContext; @@ -43,7 +44,8 @@ const EC_RESPONSE_HEADERS: &[&str] = &[ /// revocation marker. There is no separate consent KV store to clean up. /// /// `eids_cookie` should be the raw value of the `ts-eids` cookie extracted -/// from the request *before* routing consumes it. +/// from the request *before* routing consumes it. `/auction` responses ignore +/// it in favor of the request's own client set. pub fn ec_finalize_response( settings: &Settings, ec_context: &mut EcContext, @@ -81,12 +83,8 @@ pub fn ec_finalize_response( let source = ec_context.eid_sync_source(); let collected = source .map(|_| { - collect_eid_updates( - eids_cookie, - sharedid_cookie, - ec_context.client_eids(), - registry, - ) + let (cookie_input, client_input) = browser_eid_input(ec_context, eids_cookie); + collect_eid_updates(cookie_input, sharedid_cookie, client_input, registry) }) .unwrap_or_default(); let had_eid_updates = !collected.is_empty(); @@ -138,13 +136,9 @@ pub fn ec_finalize_response( return; }; + let (cookie_input, client_input) = browser_eid_input(ec_context, eids_cookie); let updates = gate_eid_updates_by_consent( - collect_eid_updates( - eids_cookie, - sharedid_cookie, - ec_context.client_eids(), - registry, - ), + collect_eid_updates(cookie_input, sharedid_cookie, client_input, registry), ec_context.consent(), ); sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, EidSyncSource::NewEc); @@ -158,6 +152,23 @@ pub fn ec_finalize_response( reconcile_pull_sync_marker(settings, registry, ec_context, response); } +/// Chooses the browser EID input KV ingestion reads for this request. +/// +/// `/auction` ingests only the client set the auction itself used (its body, +/// else its `ts-eids` cookie, as `handle_auction` resolved it). A cookie from +/// an earlier page is then never ingested next to a scoped body, and an +/// `/auction` that resolved no client set ingests nothing from the cookie. +/// Every other route, and requests with no sync source, read the cookie. +fn browser_eid_input<'a>( + ec_context: &'a EcContext, + eids_cookie: Option<&'a str>, +) -> (Option<&'a str>, Option<&'a [Eid]>) { + match ec_context.eid_sync_source() { + Some(EidSyncSource::Auction) => (None, ec_context.client_eids()), + _ => (eids_cookie, None), + } +} + fn sync_eid_cookie_updates( graph: &KvIdentityGraph, ec_context: &mut EcContext, @@ -1115,17 +1126,15 @@ mod tests { ); ec_context.set_recovery_eligible(true); ec_context.set_eid_sync_source(EidSyncSource::Navigation); - ec_context.set_client_eids(vec![Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { - id: "ID5_should_not_persist".to_owned(), - atype: Some(1), - ext: None, - }], - }]); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); + let eids_cookie = base64::engine::general_purpose::STANDARD.encode( + serde_json::to_vec(&serde_json::json!([ + { "source": "id5-sync.com", "uids": [{ "id": "ID5_should_not_persist", "atype": 1 }] } + ])) + .expect("should serialize test cookie payload"), + ); let mut response = empty_response(); ec_finalize_response( @@ -1133,7 +1142,7 @@ mod tests { &mut ec_context, Some(&graph), ®istry, - None, + Some(&eids_cookie), None, &mut response, ); @@ -1293,6 +1302,58 @@ mod tests { } } + #[test] + fn finalize_generated_ec_without_sync_source_ingests_the_ts_eids_cookie() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("genck1"); + let graph = KvIdentityGraph::in_memory("test_store"); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed generated row"); + let mut ec_context = + make_context(Some(&ec_id), None, false, true, Jurisdiction::NonRegulated); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + let registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) + .expect("should build registry"); + let eids_cookie = base64::engine::general_purpose::STANDARD.encode( + serde_json::to_vec(&serde_json::json!([ + { "source": "ids.example.com", "uids": [{ "id": "cookie-uid" }] } + ])) + .expect("should serialize test cookie payload"), + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + Some(&eids_cookie), + None, + &mut response, + ); + + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert_eq!( + stored + .ids + .iter() + .map(|(source, id)| (source.as_str(), id.uid.as_str())) + .collect::>(), + vec![("ids.example.com", "cookie-uid")], + "a new EC with no sync source should ingest the cookie" + ); + } + #[test] fn finalize_generated_ec_persists_eid_updates() { let settings = create_test_settings(); @@ -1517,22 +1578,22 @@ mod tests { .expect("should serialize test cookie payload"), ); ec_context.set_client_eids(vec![ - Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_from_body".to_owned(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_owned(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_owned(), + vec![Uid { id: "LR_from_body".to_owned(), atype: Some(3), ext: None, }], - }, + ), ]); let partners = vec![make_partner("id5-sync.com"), make_partner("liveramp.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); @@ -1595,14 +1656,14 @@ mod tests { ec_id: ec_id.clone(), }); ec_context.set_eid_sync_source(EidSyncSource::Auction); - ec_context.set_client_eids(vec![Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_should_not_persist".to_owned(), atype: Some(1), ext: None, }], - }]); + )]); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 94464d1a57..c22a592461 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -124,13 +124,15 @@ pub fn handle_identify( } } - let eid = uid.as_ref().map(|u| Eid { - source: partner.source_domain.clone(), - uids: vec![Uid { - id: u.clone(), - atype: Some(partner.openrtb_atype), - ext: None, - }], + let eid = uid.as_ref().map(|u| { + Eid::new( + partner.source_domain.clone(), + vec![Uid { + id: u.clone(), + atype: Some(partner.openrtb_atype), + ext: None, + }], + ) }); let body = IdentifyResponse { diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index f24c489edd..cc6705eaf8 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -33,6 +33,7 @@ mod auth; pub mod admin; pub mod batch_sync; +pub(crate) mod client_eids; pub mod consent; pub mod cookies; pub mod device; diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index d823cb32fb..9d2b587806 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -3,17 +3,19 @@ //! Parses the `ts-eids` cookie written by the TSJS Prebid integration and //! syncs matched partner UIDs to the KV identity graph. //! -//! The current cookie format stores a base64-encoded JSON array of full -//! OpenRTB-style `Eid` objects (`{source, uids:[...]}`). For rollout -//! compatibility we also accept the earlier flattened payload shape -//! (`{source, id, atype}` per entry). +//! The current cookie holds a base64-encoded JSON array of OpenRTB-style EID +//! objects, validated entry by entry with the same parser as the `/auction` +//! body (`ec::client_eids`). The earlier flattened payload shape +//! (`{source, id, atype}` per entry) is still accepted. use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use serde::Deserialize; use serde_json::Value as JsonValue; +use serde_json::value::RawValue; use crate::openrtb::{Eid, Uid}; +use super::client_eids::parse_client_eids; use super::kv::PartnerIdUpdate; use super::kv_types::MAX_UID_LENGTH; use super::registry::PartnerRegistry; @@ -29,26 +31,9 @@ struct LegacyCookieEid { atype: i32, } -/// OpenRTB-style `ts-eids` cookie entry. -#[derive(Debug, Deserialize)] -struct StructuredCookieEid { - source: String, - #[serde(default)] - uids: Vec, -} - -#[derive(Debug, Deserialize)] -struct StructuredCookieUid { - id: String, - #[serde(default)] - atype: Option, - #[serde(default)] - ext: Option, -} - enum DecodedCookieEids { Legacy(Vec), - Structured(Vec), + Structured(Box), } pub(crate) struct DiagnosticEidSource { @@ -91,8 +76,11 @@ fn decode_prebid_eids_cookie(cookie_value: &str) -> Result>(&bytes) + let structured = serde_json::from_slice::>(&bytes) .map_err(|e| format!("JSON parse failed: {e}"))?; + if !structured.get().trim_start().starts_with('[') { + return Err("JSON parse failed: ts-eids payload is not an array".to_owned()); + } Ok(DecodedCookieEids::Structured(structured)) } @@ -107,34 +95,59 @@ impl DecodedCookieEids { uids: vec![entry.id.clone()], }) .collect(), - Self::Structured(entries) => entries - .iter() - .filter(|entry| !entry.source.is_empty()) - .map(|entry| DiagnosticEidSource { - source: entry.source.clone(), - uids: entry.uids.iter().map(|uid| uid.id.clone()).collect(), - }) - .collect(), + Self::Structured(raw) => raw_diagnostic_sources(raw), } } fn into_openrtb(self) -> Vec { match self { Self::Legacy(entries) => legacy_cookie_eids_to_openrtb(entries), - Self::Structured(entries) => structured_cookie_eids_to_openrtb(entries), + Self::Structured(raw) => parse_client_eids(&raw), } } } +/// Reports every source and raw UID string in a structured cookie, before +/// validation, so admin diagnostics can tell "no partner" from "no valid UID". +fn raw_diagnostic_sources(raw: &RawValue) -> Vec { + let Ok(elements) = serde_json::from_str::>>(raw.get()) else { + return Vec::new(); + }; + + elements + .iter() + .filter_map(|element| { + let JsonValue::Object(entry) = serde_json::from_str::(element.get()).ok()? + else { + return None; + }; + let source = entry + .get("source") + .and_then(JsonValue::as_str) + .filter(|source| !source.is_empty())? + .to_owned(); + let uids = entry + .get("uids") + .and_then(JsonValue::as_array) + .map(|uids| { + uids.iter() + .filter_map(|uid| uid.get("id").and_then(JsonValue::as_str)) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default(); + Some(DiagnosticEidSource { source, uids }) + }) + .collect() +} + /// Collects validated request-local partner updates without performing KV I/O. /// -/// `body_eids` is the current request's own EIDs (e.g. an `/auction` JSON -/// body), when the route captured any. It is applied after the `ts-eids` -/// cookie so it wins on conflicts: the cookie is size-capped by the browser -/// (see `MAX_EID_COOKIE_BYTES` in the TSJS Prebid integration) and may be -/// missing partners the request body still carries in full. The `sharedId` -/// cookie is applied last, unaffected by `body_eids`, preserving its existing -/// override behavior. +/// Finalization passes either the `ts-eids` cookie or the request's own client +/// EIDs, never both: `/auction` passes the client set it used, every other +/// route the cookie. The `sharedId` cookie is applied last, so it wins over a +/// Prebid EID for the same partner. With repeated sources, the later entry's +/// first valid UID is the candidate for that partner. pub(crate) fn collect_eid_updates( eids_cookie: Option<&str>, sharedid_cookie: Option<&str>, @@ -277,61 +290,19 @@ fn sharedid_cookie_exceeds_size_limit(cookie_value: &str) -> bool { cookie_value.len() > MAX_UID_LENGTH } -fn structured_cookie_eids_to_openrtb(entries: Vec) -> Vec { - let mut eids = Vec::new(); - - for entry in entries { - if entry.source.is_empty() { - continue; - } - - let uids: Vec<_> = entry - .uids - .into_iter() - .filter_map(structured_cookie_uid_to_openrtb) - .collect(); - if uids.is_empty() { - continue; - } - - eids.push(Eid { - source: entry.source, - uids, - }); - } - - eids -} - -fn structured_cookie_uid_to_openrtb(uid: StructuredCookieUid) -> Option { - if uid.id.is_empty() { - return None; - } - - let atype = uid.atype.filter(|atype| *atype >= 0); - let ext = match uid.ext { - Some(JsonValue::Object(_)) => uid.ext, - _ => None, - }; - - Some(Uid { - id: uid.id, - atype, - ext, - }) -} - fn legacy_cookie_eids_to_openrtb(entries: Vec) -> Vec { entries .into_iter() .filter(|entry| !entry.source.is_empty() && !entry.id.is_empty()) - .map(|entry| Eid { - source: entry.source, - uids: vec![Uid { - id: entry.id, - atype: (entry.atype >= 0).then_some(entry.atype), - ext: None, - }], + .map(|entry| { + Eid::new( + entry.source, + vec![Uid { + id: entry.id, + atype: (entry.atype >= 0).then_some(entry.atype), + ext: None, + }], + ) }) .collect() } @@ -377,6 +348,100 @@ mod tests { BASE64.encode(serde_json::to_vec(value).expect("should serialize EID JSON")) } + #[test] + fn structured_cookie_keeps_eid_level_fields() { + let entries = json!([{ + "source": "adserver.org", + "inserter": "adserver.org", + "matcher": "adserver.org", + "mm": 4, + "uids": [{ "id": "fictional-tdid", "atype": 1 }] + }]); + + let decoded = + parse_prebid_eids_cookie(&encode_json(&entries)).expect("should decode cookie"); + + assert_eq!( + serde_json::to_value(&decoded).expect("should serialize decoded EIDs"), + entries, + "should keep EID-level fields from the cookie" + ); + } + + #[test] + fn structured_cookie_drops_a_mistyped_field_instead_of_the_whole_cookie() { + let encoded = encode_json(&json!([ + { "source": "a.example.com", "uids": [{ "id": "a-uid", "atype": "1" }] }, + { "source": "b.example.com", "uids": [{ "id": "b-uid", "atype": 3 }] } + ])); + + let decoded = parse_prebid_eids_cookie(&encoded).expect("should decode cookie"); + + assert_eq!( + serde_json::to_value(&decoded).expect("should serialize decoded EIDs"), + json!([ + { "source": "a.example.com", "uids": [{ "id": "a-uid" }] }, + { "source": "b.example.com", "uids": [{ "id": "b-uid", "atype": 3 }] } + ]), + "a mistyped atype should drop only that field" + ); + } + + #[test] + fn structured_cookie_isolates_an_unparsable_entry() { + let raw = r#"[{"source":"a.example.com","uids":[{"id":"a-uid"}]},{"source":"bad.example.com","uids":[{"id":"bad-\ud800"}]}]"#; + + let decoded = parse_prebid_eids_cookie(&BASE64.encode(raw)).expect("should decode cookie"); + + assert_eq!( + serde_json::to_value(&decoded).expect("should serialize decoded EIDs"), + json!([{ "source": "a.example.com", "uids": [{ "id": "a-uid" }] }]), + "a lone surrogate should cost only its own entry" + ); + } + + #[test] + fn structured_cookie_applies_the_entry_cap() { + let entries: Vec = (0..65) + .map(|index| json!({ "source": format!("s{index}.example.com"), "uids": [{ "id": "u" }] })) + .collect(); + + let decoded = + parse_prebid_eids_cookie(&encode_json(&serde_json::Value::Array(entries.clone()))) + .expect("should decode cookie"); + + assert_eq!( + serde_json::to_value(&decoded).expect("should serialize decoded EIDs"), + serde_json::Value::Array(entries[..64].to_vec()), + "the body's 64-entry cap should apply to the cookie" + ); + } + + #[test] + fn structured_cookie_diagnostics_report_raw_uid_strings() { + let registry = make_registry(vec![("example", "example.com")]); + let oversized = "x".repeat(MAX_UID_LENGTH + 1); + let encoded = + encode_json(&json!([{ "source": "example.com", "uids": [{ "id": oversized }] }])); + + let analysis = + analyze_prebid_eids_cookie(&encoded, ®istry).expect("should decode cookie"); + + assert!( + analysis.eids.is_empty(), + "an oversized UID should not survive validation" + ); + assert_eq!( + analysis + .diagnostic_sources + .iter() + .map(|source| (source.source.clone(), source.uids.clone())) + .collect::>(), + vec![("example.com".to_owned(), vec![oversized])], + "diagnostics should still report the raw source and UID so admin can say no valid UID" + ); + } + #[test] fn parse_prebid_eids_cookie_parses_legacy_flat_payload() { let eids = vec![ @@ -463,6 +528,16 @@ mod tests { assert!(result.is_err(), "should reject invalid JSON"); } + #[test] + fn parse_prebid_eids_cookie_rejects_non_array_json() { + let result = parse_prebid_eids_cookie(&encode_json(&json!({"a": 1}))); + let error = result.expect_err("should reject a JSON object"); + assert!( + error.contains("not an array"), + "should explain that the payload is not an array" + ); + } + #[test] fn ts_eids_cookie_rejects_oversized_payloads() { let oversized = "x".repeat(MAX_EIDS_COOKIE_BYTES + 1); @@ -636,22 +711,22 @@ mod tests { {"source": "id5-sync.com", "uids": [{"id": "ID5_stale", "atype": 1}]} ])); let body_eids = vec![ - Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_fresh".to_owned(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_owned(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_owned(), + vec![Uid { id: "LR_xyz".to_owned(), atype: Some(3), ext: None, }], - }, + ), ]; let updates = collect_eid_updates(Some(&eids_cookie), None, Some(&body_eids), ®istry); diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 4f45fbeb9c..51f6bc7804 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -1969,7 +1969,10 @@ pub fn register_providers( #[cfg(test)] mod tests { use super::*; - use crate::auction::test_support::canonical_parity_auction_request; + use crate::auction::test_support::{ + canonical_parity_auction_request, repeated_source_provenance_eids, + repeated_source_provenance_eids_json, + }; use crate::auction::types::{ AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, PublisherInfo, UserInfo, }; @@ -2305,6 +2308,36 @@ mod tests { ); } + #[test] + fn aps_request_forwards_eid_provenance_fields_and_repeated_sources() { + let provider = ApsAuctionProvider::new(config()); + let mut auction_request = canonical_parity_auction_request(); + auction_request.user.eids = Some(repeated_source_provenance_eids()); + let settings = create_test_settings(); + let services = noop_services(); + let downstream = http::Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build downstream request"); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 321, + transport_timeout_ms: 321, + provider_responses: None, + services: &services, + }; + + let openrtb = provider.build_openrtb_request(&auction_request, &context); + + let serialized = serde_json::to_value(&openrtb).expect("should serialize request"); + assert_eq!( + serialized["user"]["ext"]["eids"], + repeated_source_provenance_eids_json(), + "APS should receive both entries with their own EID-level fields" + ); + } + #[test] fn builds_aps_openrtb_request_with_explicit_privacy_policy() { let provider = ApsAuctionProvider::new(config()); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index f1d1d0b509..a7db8624e4 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -3590,6 +3590,7 @@ mod tests { use crate::auction::test_support::{ canonical_parity_auction_request, create_test_auction_context as shared_test_auction_context, + repeated_source_provenance_eids, repeated_source_provenance_eids_json, }; use crate::auction::types::{ AdFormat, AdSlot, AuctionContext, AuctionRequest, DeviceInfo, PublisherInfo, UserInfo, @@ -6858,35 +6859,59 @@ external_bundle_sri = "sha384-AAAA" ); } + #[test] + fn to_openrtb_forwards_eid_provenance_fields_and_repeated_sources() { + let provider = PrebidAuctionProvider::new(base_config()); + let mut auction_request = create_test_auction_request(); + auction_request.user.eids = Some(repeated_source_provenance_eids()); + let settings = make_settings(); + let request = build_test_request(); + let context = create_test_auction_context(&settings, &request); + + let openrtb = provider.to_openrtb( + &auction_request, + &context, + None, + make_request_info(&context), + ); + + let serialized = serde_json::to_value(&openrtb).expect("should serialize OpenRTB request"); + assert_eq!( + serialized["user"]["ext"]["eids"], + repeated_source_provenance_eids_json(), + "Prebid Server should receive both entries with their own EID-level fields" + ); + } + #[test] fn to_openrtb_includes_eids_from_auction_request() { let provider = PrebidAuctionProvider::new(base_config()); let mut auction_request = create_test_auction_request(); auction_request.user.eids = Some(vec![ - crate::openrtb::Eid { - source: "liveramp.com".to_owned(), - uids: vec![crate::openrtb::Uid { + crate::openrtb::Eid::new( + "liveramp.com".to_owned(), + vec![crate::openrtb::Uid { id: "LR_xyz".to_owned(), atype: Some(3), ext: None, }], - }, - crate::openrtb::Eid { - source: "id5-sync.com".to_owned(), - uids: vec![crate::openrtb::Uid { + ), + crate::openrtb::Eid::new( + "id5-sync.com".to_owned(), + vec![crate::openrtb::Uid { id: "ID5_abc".to_owned(), atype: Some(1), ext: None, }], - }, - crate::openrtb::Eid { - source: "google.com".to_owned(), - uids: vec![crate::openrtb::Uid { + ), + crate::openrtb::Eid::new( + "google.com".to_owned(), + vec![crate::openrtb::Uid { id: "pair-id".to_owned(), atype: Some(571187), ext: None, }], - }, + ), ]); let settings = make_settings(); diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 17b3d624f6..683fbbc138 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -75,6 +75,7 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("creative.rs", include_str!("creative.rs")), ("ec/auth.rs", include_str!("ec/auth.rs")), ("ec/batch_sync.rs", include_str!("ec/batch_sync.rs")), + ("ec/client_eids.rs", include_str!("ec/client_eids.rs")), ("ec/consent.rs", include_str!("ec/consent.rs")), ("ec/cookies.rs", include_str!("ec/cookies.rs")), ("ec/device.rs", include_str!("ec/device.rs")), diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 4aded74882..086bb6ce8f 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -65,17 +65,62 @@ pub struct ConsentedProvidersSettings { } /// An Extended User ID entry from an identity provider. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// +/// Mirrors `OpenRTB` 2.6 §3.2.27. Browser entries keep the EID-level fields +/// Prebid set; KV-resolved entries leave them empty. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Eid { /// Identity provider domain (e.g. `"id5-sync.com"`). pub source: String, /// One or more user IDs from this provider. pub uids: Vec, + /// Entity that inserted the IDs into the request (`eid.inserter`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option, + /// Entity that matched the IDs (`eid.matcher`). Absent means `source`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option, + /// Match method (`eid.mm`). Vendor values from 500 up are passed through. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// Provider-specific extension data, e.g. `LiveIntent` `segments`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, +} + +impl Eid { + /// Builds an [`Eid`] without EID-level fields. + /// + /// Use this for EIDs that never carry provenance, such as KV-resolved + /// partner IDs. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::openrtb::{Eid, Uid}; + /// + /// let eid = Eid::new( + /// "ids.example.com".to_owned(), + /// vec![Uid { id: "uid-1".to_owned(), atype: Some(1), ext: None }], + /// ); + /// assert_eq!(eid.inserter, None); + /// ``` + #[must_use] + pub fn new(source: String, uids: Vec) -> Self { + Self { + source, + uids, + inserter: None, + matcher: None, + mm: None, + ext: None, + } + } } /// A single user identifier within an [`Eid`] entry. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Uid { /// The identifier value. @@ -194,6 +239,87 @@ impl ToExt for ResponseExt {} mod tests { use super::*; use crate::auction::types::OrchestratorExt; + use serde_json::json; + + #[test] + fn eid_serializes_openrtb_provenance_fields_when_present() { + let eid = Eid { + source: "ids.example.com".to_owned(), + uids: vec![Uid { + id: "uid-1".to_owned(), + atype: Some(1), + ext: Some(json!({ "rtiPartner": "TDID" })), + }], + inserter: Some("inserter.example.com".to_owned()), + matcher: Some("matcher.example.com".to_owned()), + mm: Some(4), + ext: Some(json!({ "segments": ["s1"] })), + }; + + let serialized = serde_json::to_value(&eid).expect("should serialize EID"); + + assert_eq!( + serialized, + json!({ + "source": "ids.example.com", + "uids": [{ "id": "uid-1", "atype": 1, "ext": { "rtiPartner": "TDID" } }], + "inserter": "inserter.example.com", + "matcher": "matcher.example.com", + "mm": 4, + "ext": { "segments": ["s1"] } + }), + "should serialize every EID-level field" + ); + } + + #[test] + fn eid_without_provenance_fields_serializes_as_before() { + let eid = Eid::new( + "ids.example.com".to_owned(), + vec![Uid { + id: "uid-1".to_owned(), + atype: Some(1), + ext: None, + }], + ); + + let serialized = serde_json::to_string(&eid).expect("should serialize EID"); + + assert_eq!( + serialized, r#"{"source":"ids.example.com","uids":[{"id":"uid-1","atype":1}]}"#, + "an EID without EID-level fields should serialize byte-identically to before" + ); + } + + #[test] + fn eid_deserializes_provenance_fields_and_keeps_rejecting_unknown_keys() { + let parsed: Eid = serde_json::from_str( + r#"{"source":"ids.example.com","uids":[{"id":"uid-1"}],"inserter":"inserter.example.com","matcher":"matcher.example.com","mm":3,"ext":{"k":"v"}}"#, + ) + .expect("should deserialize EID-level fields"); + + assert_eq!( + parsed, + Eid { + source: "ids.example.com".to_owned(), + uids: vec![Uid { + id: "uid-1".to_owned(), + atype: None, + ext: None, + }], + inserter: Some("inserter.example.com".to_owned()), + matcher: Some("matcher.example.com".to_owned()), + mm: Some(3), + ext: Some(json!({ "k": "v" })), + }, + "should read every EID-level field" + ); + assert!( + serde_json::from_str::(r#"{"source":"ids.example.com","uids":[],"unknown":1}"#) + .is_err(), + "should keep deny_unknown_fields" + ); + } #[test] fn openrtb_response_round_trips_with_struct_literals() { @@ -393,14 +519,14 @@ mod tests { #[test] fn eid_serializes_correctly() { - let eid = Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + let eid = Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5-abc123".to_owned(), atype: Some(1), ext: None, }], - }; + ); let serialized = serde_json::to_value(&eid).expect("should serialize"); assert_eq!(serialized["source"], "id5-sync.com", "source should match"); @@ -417,14 +543,14 @@ mod tests { #[test] fn eid_serializes_vendor_specific_atype() { - let eid = Eid { - source: "google.com".to_owned(), - uids: vec![Uid { + let eid = Eid::new( + "google.com".to_owned(), + vec![Uid { id: "pair-id".to_owned(), atype: Some(571187), ext: None, }], - }; + ); let serialized = serde_json::to_value(&eid).expect("should serialize"); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 682ecc0f15..97211feb84 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -7490,10 +7490,15 @@ mod tests { ..marker_only })); } + use crate::auction::test_support::canonical_parity_auction_request; use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::consent::ConsentContext; + use crate::consent::jurisdiction::Jurisdiction; + use crate::cookies::parse_cookies_to_jar; + use crate::ec::EcKvSnapshot; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteOutcome}; + use crate::ec::kv_types::{KvEntry, KvPartnerId}; use crate::integrations::IntegrationRegistry; use crate::platform::test_support::{ NoopSecretStore, StubHttpClient, build_services_with_http_client, @@ -7501,6 +7506,7 @@ mod tests { noop_services_with_telemetry_sink, }; use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; + use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use edgezero_core::body::Body as EdgeBody; use http::{Method, Request as HttpRequest, StatusCode, header}; use std::sync::Arc; @@ -7887,6 +7893,80 @@ mod tests { } } + #[test] + fn body_less_auction_eids_put_cookie_entries_first_and_kv_only_sources_after() { + let ec_id = format!("{}.navmrg", "a".repeat(64)); + let registry = PartnerRegistry::from_config(&[ + marker_partner("ids.example.com"), + marker_partner("other.example.com"), + ]) + .expect("should build partner registry"); + let mut entry = KvEntry::minimal("ids.example.com", "kv-stale", 1000); + entry.ids.insert( + "other.example.com".to_owned(), + KvPartnerId { + uid: "kv-other".to_owned(), + }, + ); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(entry), + generation: None, + }; + let ec_context = EcContext::new_for_test( + Some(ec_id.clone()), + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..ConsentContext::default() + }, + ); + let cookie_entries = serde_json::json!([{ + "source": "ids.example.com", + "inserter": "inserter-1.example.com", + "mm": 1, + "uids": [{ "id": "browser-fresh", "atype": 1 }] + }]); + let cookie = + BASE64.encode(serde_json::to_vec(&cookie_entries).expect("should serialize cookie")); + let jar = parse_cookies_to_jar(&format!("{COOKIE_TS_EIDS}={cookie}")); + let services = noop_services(); + let mut auction_request = canonical_parity_auction_request(); + // No TCF signal and no GDPR, so the consent gate passes the merged set. + auction_request.user.consent = None; + auction_request.user.eids = None; + + apply_auction_eids_and_device( + &mut auction_request, + &AuctionEidTargeting { + cookie_jar: Some(&jar), + ec_id: Some(ec_id.as_str()), + kv_snapshot: &snapshot, + partner_registry: Some(®istry), + ec_context: &ec_context, + services: &services, + geo: None, + path_label: "Test", + }, + ); + + assert_eq!( + serde_json::to_value(&auction_request.user.eids).expect("should serialize EIDs"), + serde_json::json!([ + { + "source": "ids.example.com", + "inserter": "inserter-1.example.com", + "mm": 1, + "uids": [{ "id": "browser-fresh", "atype": 1 }] + }, + { + "source": "other.example.com", + "uids": [{ "id": "kv-other", "atype": crate::settings::EcPartner::default_openrtb_atype() }] + } + ]), + "navigation and page-bids auctions should forward the cookie entry as sent, then KV-only sources" + ); + } + async fn run_marker_lookup_probe( probe: MarkerProbe, has_eid_cookie: bool, diff --git a/crates/trusted-server-js/lib/src/core/auction.ts b/crates/trusted-server-js/lib/src/core/auction.ts index a026843624..0bd2f16b73 100644 --- a/crates/trusted-server-js/lib/src/core/auction.ts +++ b/crates/trusted-server-js/lib/src/core/auction.ts @@ -20,18 +20,14 @@ export interface AdRequestUnit { bids: Array<{ bidder: string; params: Record }>; } -/** A user identifier within an auction-level EID entry. */ -export interface AuctionUid { - id: string; - atype?: number; - ext?: Record; -} - -/** An auction-level EID entry forwarded to the server. */ -export interface AuctionEid { - source: string; - uids: AuctionUid[]; -} +/** + * An auction-level EID entry forwarded to the server. + * + * TSJS forwards the entries Prebid placed in the `trustedServer` bidder + * request as plain data and the server validates each one, so the server + * defines the contract. + */ +export type AuctionEid = unknown; /** The payload POSTed to the /auction orchestrator. */ export interface AdRequest { diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/eids.ts b/crates/trusted-server-js/lib/src/integrations/prebid/eids.ts new file mode 100644 index 0000000000..fd25b2e689 --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/prebid/eids.ts @@ -0,0 +1,226 @@ +// Plain-data copies of the EIDs Prebid placed in the `trustedServer` bidder +// request, and the `ts-eids` cookie value built from them. +// +// Pure functions: nothing here reads `pbjs`, the DOM or the clock. The server +// validates every entry of the body and the cookie alike (spec 4.4), so this +// module checks nothing beyond what serialization needs. + +/** UTF-8 byte budget for the EIDs forwarded in one `/auction` body. */ +export const MAX_AUCTION_EIDS_BYTES = 65_536; + +/** Maximum length of the base64 `ts-eids` cookie value. */ +export const MAX_EID_COOKIE_BYTES = 3072; + +/** Counts reported by {@link collectBidderEids} for debug logging. */ +export interface EidCollectionStats { + /** Entries copied into the result. */ + forwarded: number; + /** Entries dropped because they could not be serialized. */ + unserializable: number; + /** Entries dropped because the byte budget was exhausted. */ + overBudget: number; +} + +/** A fitted `ts-eids` cookie value and the sources trimming touched. */ +export interface FittedEidCookie { + /** Base64 cookie value, or `undefined` when nothing fits. */ + value: string | undefined; + /** Sources removed whole. */ + droppedSources: string[]; + /** Sources that lost UIDs but stayed in the cookie. */ + trimmedSources: string[]; +} + +const utf8Encoder = new TextEncoder(); + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** + * Returns the arrays Prebid may hold EIDs in, in reading order: + * `ortb2.user.ext.eids`, then `ortb2.user.eids`. RTD modules write after + * Prebid normalizes first-party data, so `user.eids` can hold entries too. + * Prebid redacts both paths for the bidder, so reading both does not widen + * what `trustedServer` receives. + */ +function bidderEidArrays(bidderRequest: unknown): unknown[][] { + if (!isRecord(bidderRequest) || !isRecord(bidderRequest.ortb2)) { + return []; + } + const user = bidderRequest.ortb2.user; + if (!isRecord(user)) { + return []; + } + const arrays: unknown[][] = []; + if (isRecord(user.ext) && Array.isArray(user.ext.eids)) { + arrays.push(user.ext.eids); + } + if (Array.isArray(user.eids)) { + arrays.push(user.eids); + } + return arrays; +} + +function serializeEntry(entry: unknown): string | undefined { + try { + // `undefined` and functions serialize to `undefined`; treat them as + // unserializable like a circular object or a BigInt. + return JSON.stringify(entry); + } catch { + return undefined; + } +} + +/** + * Copies the EIDs Prebid placed in a `trustedServer` bidder request. + * + * Reads `ortb2.user.ext.eids` then `ortb2.user.eids`, copies each entry + * through a JSON round trip so the result is plain data (`undefined` fields + * disappear), drops entries that cannot be serialized, and keeps entries in + * order while their serializations total at most + * {@link MAX_AUCTION_EIDS_BYTES} UTF-8 bytes, dropping the rest. Entries are + * not merged and not validated. + * + * @param bidderRequest the second `buildRequests` argument + * @param stats filled with counts for debug logging, when given + * @returns the copied entries, or `undefined` when none remain + */ +export function collectBidderEids( + bidderRequest: unknown, + stats?: EidCollectionStats +): unknown[] | undefined { + const collected: unknown[] = []; + let usedBytes = 0; + let budgetExhausted = false; + let unserializable = 0; + let overBudget = 0; + + for (const entries of bidderEidArrays(bidderRequest)) { + for (const entry of entries) { + const serialized = serializeEntry(entry); + if (serialized === undefined) { + unserializable += 1; + continue; + } + const bytes = utf8Encoder.encode(serialized).length; + if (budgetExhausted || usedBytes + bytes > MAX_AUCTION_EIDS_BYTES) { + budgetExhausted = true; + overBudget += 1; + continue; + } + usedBytes += bytes; + collected.push(JSON.parse(serialized)); + } + } + + if (stats) { + stats.forwarded = collected.length; + stats.unserializable = unserializable; + stats.overBudget = overBudget; + } + return collected.length > 0 ? collected : undefined; +} + +function withoutEidExt(entry: unknown): unknown { + if (!isRecord(entry) || !('ext' in entry)) { + return entry; + } + const copy: Record = {}; + for (const [key, value] of Object.entries(entry)) { + if (key !== 'ext') { + copy[key] = value; + } + } + return copy; +} + +/** JSON with object keys sorted at every level, for equality checks. */ +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(',')}]`; + } + if (isRecord(value)) { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) + .join(',')}}`; + } + return JSON.stringify(value) ?? 'null'; +} + +function cookieLabel(entry: unknown): string { + return isRecord(entry) && typeof entry.source === 'string' ? entry.source : '(malformed entry)'; +} + +function encodeUtf8Base64(text: string): string { + let binary = ''; + for (const byte of utf8Encoder.encode(text)) { + binary += String.fromCharCode(byte); + } + return btoa(binary); +} + +/** + * Builds the `ts-eids` cookie value from a request's collected EIDs. + * + * Removes each entry's EID-level `ext`, drops entries that become identical + * (compared with sorted keys), then trims from the tail until the standard + * padded base64 of the JSON array's UTF-8 bytes is at most + * {@link MAX_EID_COOKIE_BYTES} long: drop UIDs from the last entry, then the + * entry itself, and repeat. A last entry that is not an object with a `uids` + * array is removed whole. The input is not modified. + * + * @param eids the entries forwarded in the `/auction` body + * @returns the cookie value, or `undefined` when nothing fits, and the + * sources trimming touched + */ +export function fitEidCookie(eids: readonly unknown[]): FittedEidCookie { + const seen = new Set(); + const payload: unknown[] = []; + for (const entry of eids) { + const stripped = withoutEidExt(entry); + const key = canonicalJson(stripped); + if (seen.has(key)) { + continue; + } + seen.add(key); + payload.push( + isRecord(stripped) && Array.isArray(stripped.uids) + ? { ...stripped, uids: [...stripped.uids] } + : stripped + ); + } + + const dropped = new Set(); + const trimmed = new Set(); + while (payload.length > 0) { + const value = encodeUtf8Base64(JSON.stringify(payload)); + if (value.length <= MAX_EID_COOKIE_BYTES) { + return { + value, + droppedSources: [...dropped], + trimmedSources: [...trimmed], + }; + } + + const last = payload[payload.length - 1]; + if (isRecord(last) && Array.isArray(last.uids) && last.uids.length > 1) { + last.uids = last.uids.slice(0, -1); + trimmed.add(cookieLabel(last)); + continue; + } + + payload.pop(); + const label = cookieLabel(last); + dropped.add(label); + // A source trimmed and then dropped is reported once, as dropped. + trimmed.delete(label); + } + + return { + value: undefined, + droppedSources: [...dropped], + trimmedSources: [...trimmed], + }; +} diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 82c0e8c6b7..aec08160bb 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -25,24 +25,23 @@ import { import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import { registerApsPrebidRenderer, validateApsRenderer } from '../aps/render'; -import type { AuctionBid, AuctionEid } from '../../core/auction'; +import type { AuctionBid } from '../../core/auction'; import type { AuctionSlot, TsjsApi } from '../../core/types'; +import { + collectBidderEids, + fitEidCookie, + MAX_EID_COOKIE_BYTES, + type EidCollectionStats, +} from './eids'; import { PREBID_USER_ID_MODULE_REGISTRY, userIdConfigNameAliases, userIdSubmoduleKey, } from './user_id_modules'; -/** - * Prebid.js public API surface (type-only; erased at build time). - * - * `getUserIdsAsEids` is added by the userId module at runtime, which the base - * package typing does not model. - */ -type PbjsGlobal = typeof _pbjsDefault & { - getUserIdsAsEids?: () => unknown[]; -}; +/** Prebid.js public API surface (type-only; erased at build time). */ +type PbjsGlobal = typeof _pbjsDefault; // Prebid.js itself is NOT bundled into this module. It is served as the // external bundle configured via `integrations.prebid.external_bundle_url` @@ -148,9 +147,6 @@ const APS_BIDDER_CODE = 'aps'; // both after the registration attempt. const APS_RENDERER_FIELD = 'trustedServerRenderer'; const APS_BID_RESPONSE_LISTENER_SENTINEL = '__tsApsBidResponseListenerInstalled'; -// OpenRTB permits vendor-specific agent types; PAIR uses 571187. -// Keep this range aligned with the signed 32-bit Rust/OpenRTB representation. -const MAX_OPENRTB_ATYPE = 2_147_483_647; const BIDDER_PARAMS_KEY = 'bidderParams'; const STORED_REQUEST_KEY = 'storedRequest'; const ZONE_KEY = 'zone'; @@ -1009,11 +1005,6 @@ type TrustedServerRequest = { tsjsBidRequests: TrustedServerBidRequest[]; }; -type PrebidUserIdEid = { - source?: unknown; - uids?: Array<{ id?: unknown; atype?: unknown; ext?: unknown }>; -}; - type RefreshGptSlot = { getSlotElementId?: () => string; getAdUnitPath?: () => string; @@ -1073,33 +1064,6 @@ const DEFAULT_REFRESH_SIZES: BannerSize[] = [ [300, 250], ]; -function sanitizeAuctionUid(uid: { - id?: unknown; - atype?: unknown; - ext?: unknown; -}): AuctionEid['uids'][number] | undefined { - if (typeof uid?.id !== 'string' || uid.id.length === 0) { - return undefined; - } - - const sanitizedUid: AuctionEid['uids'][number] = { id: uid.id }; - - if ( - typeof uid.atype === 'number' && - Number.isInteger(uid.atype) && - uid.atype >= 0 && - uid.atype <= MAX_OPENRTB_ATYPE - ) { - sanitizedUid.atype = uid.atype; - } - - if (uid.ext && typeof uid.ext === 'object' && !Array.isArray(uid.ext)) { - sanitizedUid.ext = uid.ext as Record; - } - - return sanitizedUid; -} - function isDefined(value: T | undefined): value is T { return value !== undefined; } @@ -1888,31 +1852,6 @@ function removePublisherState(adUnitCode?: string | string[]): void { } } -function collectAuctionEids(): AuctionEid[] | undefined { - if (typeof pbjs.getUserIdsAsEids !== 'function') { - return undefined; - } - - const rawEids = (pbjs.getUserIdsAsEids() ?? []) as PrebidUserIdEid[]; - const eids: AuctionEid[] = []; - - for (const eid of rawEids) { - if (typeof eid?.source !== 'string' || eid.source.length === 0) { - continue; - } - - const uids = Array.isArray(eid.uids) ? eid.uids.map(sanitizeAuctionUid).filter(isDefined) : []; - - if (uids.length === 0) { - continue; - } - - eids.push({ source: eid.source, uids }); - } - - return eids.length > 0 ? eids : undefined; -} - /** * Install the Prebid integration. * @@ -2354,14 +2293,13 @@ export function installPrebidNpm(config?: Partial): typeof pbjs return true; // All requests are valid — orchestrator handles filtering }, - buildRequests(validBidRequests: TrustedServerBidRequest[]): TrustedServerRequest { + buildRequests( + validBidRequests: TrustedServerBidRequest[], + bidderRequest?: unknown + ): TrustedServerRequest { log.debug('[tsjs-prebid] buildRequests', { count: validBidRequests.length }); const requestScopedBidRequests = [...validBidRequests]; - const hasUserIdApi = typeof pbjs.getUserIdsAsEids === 'function'; - const auctionEids = collectAuctionEids(); - if (hasUserIdApi && !auctionEids) { - clearPrebidEidsCookie(); - } + const auctionEids = forwardBidderEids(bidderRequest); const payload = buildAdRequest(validBidRequests, { eids: auctionEids }); return { method: 'POST', @@ -2388,6 +2326,22 @@ export function installPrebidNpm(config?: Partial): typeof pbjs }, }); + // Prebid never calls buildRequests for a bidder its TCF rules block, so + // without this a ts-eids cookie written before consent changed would live + // for its full day. Prebid emits the event at AUCTION_END, before + // bidsBackHandler. + const prebidEvents = pbjs as unknown as { + onEvent?: (event: string, handler: (payload: unknown) => void) => void; + }; + if (typeof prebidEvents.onEvent === 'function') { + prebidEvents.onEvent('tcf2Enforcement', (payload) => { + if (tcfBlocksTrustedServer(payload)) { + clearPrebidEidsCookie(); + log.debug('[tsjs-prebid] cleared ts-eids: Prebid TCF rules blocked trustedServer'); + } + }); + } + const originalRequestBids = pbjs.requestBids.bind(pbjs); // Browser demand ownership is explicit. Only validated auction-plan route @@ -2534,11 +2488,9 @@ export function installPrebidNpm(config?: Partial): typeof pbjs }, }; - // Chain a bidsBackHandler to collect Prebid User ID Module EIDs - // and persist them as a cookie for backend sync. + // Chain a bidsBackHandler to register this auction's publisher bids before the publisher's handler runs. const originalBidsBack = opts.bidsBackHandler; opts.bidsBackHandler = function (...args: unknown[]) { - syncPrebidEidsCookie(); const registrationId = isSyntheticRefresh ? undefined : registerPendingPublisherBids(publisherAdUnitCodes, args[0], firstImpressionTokens); @@ -2871,103 +2823,43 @@ export function installRefreshHandler(timeoutMs = 1500): void { }); } -/** - * Configure identity sync behavior for the generated Prebid User ID modules. - * - * The external bundle generator statically imports the selected modules into - * its generated entry. This post-window-load configuration controls when - * those modules synchronize identities; it does not select or register modules. - */ -export function installUserIdModules(): void { - try { - pbjs.setConfig({ - userSync: { - syncEnabled: true, - filterSettings: { - all: { bidders: '*', filter: 'include' }, - }, - auctionDelay: 0, - syncsPerBidder: 5, - syncDelay: 3000, - }, - }); - log.info('[tsjs-prebid] userID modules configured'); - } catch { - // pbjs not ready — userID modules will use defaults - } -} - // --------------------------------------------------------------------------- -// Prebid EID cookie sync +// Prebid EID forwarding and the ts-eids cookie // --------------------------------------------------------------------------- -/** Maximum cookie payload size in bytes (leave room for other cookies). */ -const MAX_EID_COOKIE_BYTES = 3072; - /** Cookie name for persisted Prebid EIDs. */ const EID_COOKIE_NAME = 'ts-eids'; /** Cookie max-age in seconds (1 day). */ const EID_COOKIE_MAX_AGE = 86400; +/** What `buildRequests` did with `ts-eids`, for debug logs. */ +type EidCookieDecision = 'written' | 'trimmed' | 'cleared'; + /** Clears any previously persisted Prebid EIDs cookie. */ function clearPrebidEidsCookie(): void { - document.cookie = `${EID_COOKIE_NAME}=; Path=/; Secure; SameSite=Lax; Max-Age=0`; -} - -/** - * Trims an EID payload so its base64-encoded cookie fits `MAX_EID_COOKIE_BYTES`, - * dropping UIDs then whole sources from the tail. `/auction` requests still - * forward the untrimmed set in the request body (see `buildAdRequest`); this - * cap only bounds what the `ts-eids` cookie carries for routes without a body - * (e.g. `GET /_ts/page-bids`) and for backend ingestion at response finalize. - */ -function fitAuctionEidsToCookie(eids: AuctionEid[]): AuctionEid[] | undefined { - const payload = eids.map((eid) => ({ source: eid.source, uids: [...eid.uids] })); - const droppedSources = new Set(); - const trimmedSources = new Set(); - - while (payload.length > 0) { - const encoded = btoa(JSON.stringify(payload)); - if (encoded.length <= MAX_EID_COOKIE_BYTES) { - logTrimmedEidSources(droppedSources, trimmedSources); - return payload; - } - - const last = payload[payload.length - 1]; - if (last && last.uids.length > 1) { - last.uids = last.uids.slice(0, last.uids.length - 1); - trimmedSources.add(last.source); - continue; - } - - const dropped = payload.pop(); - if (dropped) { - droppedSources.add(dropped.source); - // A source trimmed and then dropped is reported once, as dropped. - trimmedSources.delete(dropped.source); - } + try { + document.cookie = `${EID_COOKIE_NAME}=; Path=/; Secure; SameSite=Lax; Max-Age=0`; + } catch { + log.warn('[tsjs-prebid] failed to clear the ts-eids cookie'); } - - logTrimmedEidSources(droppedSources, trimmedSources); - return undefined; } /** - * Logs which EID sources `fitAuctionEidsToCookie` had to drop or trim UIDs - * from, if any. Debug level: `/auction` sends the untrimmed set in the request - * body, so a trimmed cookie is expected steady state rather than a warning. + * Logs which EID sources the cookie had to drop or trim UIDs from, if any. + * Debug level: `/auction` sends the untrimmed set in the request body, so a + * trimmed cookie is expected steady state rather than a warning. */ -function logTrimmedEidSources(droppedSources: Set, trimmedSources: Set): void { - if (droppedSources.size === 0 && trimmedSources.size === 0) { +function logTrimmedEidSources(droppedSources: string[], trimmedSources: string[]): void { + if (droppedSources.length === 0 && trimmedSources.length === 0) { return; } const parts: string[] = []; - if (droppedSources.size > 0) { - parts.push(`dropped sources: ${[...droppedSources].join(', ')}`); + if (droppedSources.length > 0) { + parts.push(`dropped sources: ${droppedSources.join(', ')}`); } - if (trimmedSources.size > 0) { - parts.push(`trimmed uids from sources: ${[...trimmedSources].join(', ')}`); + if (trimmedSources.length > 0) { + parts.push(`trimmed uids from sources: ${trimmedSources.join(', ')}`); } log.debug( `[tsjs-prebid] ts-eids cookie exceeded ${MAX_EID_COOKIE_BYTES} bytes; ${parts.join('; ')}` @@ -2975,37 +2867,77 @@ function logTrimmedEidSources(droppedSources: Set, trimmedSources: Set 0 || fitted.trimmedSources.length > 0 + ? 'trimmed' + : 'written'; + } catch { + log.warn('[tsjs-prebid] failed to write the ts-eids cookie; clearing it'); + clearPrebidEidsCookie(); + return 'cleared'; + } +} - const payload = fitAuctionEidsToCookie(eids); - if (!payload) { - clearPrebidEidsCookie(); - return; - } +/** + * Collects the EIDs Prebid placed in this `trustedServer` bidder request and + * mirrors them into `ts-eids` synchronously, before Prebid sends the request, + * so the server never falls back to a stale cookie for it. + * + * `bidderRequest.ortb2` is what Prebid lets `trustedServer` see: global + * first-party data merged with its own bidder fragment, IDs restricted to + * other bidders excluded, activity controls applied. Nothing else is read: + * not `getUserIdsAsEids()`, not the `ortb2` config, not `auctionId`. + */ +function forwardBidderEids(bidderRequest: unknown): unknown[] | undefined { + if (bidderRequest === null || typeof bidderRequest !== 'object') { + log.debug( + '[tsjs-prebid] buildRequests got no bidder request; sending no EIDs and leaving ts-eids unchanged' + ); + return undefined; + } - const encoded = btoa(JSON.stringify(payload)); - document.cookie = `${EID_COOKIE_NAME}=${encoded}; Path=/; Secure; SameSite=Lax; Max-Age=${EID_COOKIE_MAX_AGE}`; + const stats: EidCollectionStats = { + forwarded: 0, + unserializable: 0, + overBudget: 0, + }; + const eids = collectBidderEids(bidderRequest, stats); + let cookie: EidCookieDecision; + if (eids) { + cookie = writePrebidEidsCookie(eids); + } else { + clearPrebidEidsCookie(); + cookie = 'cleared'; + } + log.debug('[tsjs-prebid] forwarding bidder EIDs', { ...stats, cookie }); + return eids; +} - log.debug(`[tsjs-prebid] synced ${payload.length} EID sources to cookie`); - } catch (err) { - log.warn('[tsjs-prebid] failed to sync EIDs cookie', err); +/** Whether a `tcf2Enforcement` payload says Prebid blocked `trustedServer` or its EIDs. */ +function tcfBlocksTrustedServer(payload: unknown): boolean { + if (payload === null || typeof payload !== 'object') { + return false; } + const { biddersBlocked, eidsBlocked } = payload as { + biddersBlocked?: unknown; + eidsBlocked?: unknown; + }; + return [biddersBlocked, eidsBlocked].some( + (blocked) => Array.isArray(blocked) && blocked.includes(ADAPTER_CODE) + ); } // Self-initialize when loaded in a browser (same pattern as other integrations). @@ -3017,21 +2949,10 @@ if (typeof window !== 'undefined') { // fail to run the replacement auction — leave GPT untouched instead. if (hasPrebidJsApi()) { installRefreshHandler(); - // The slim-Prebid lazy loader appends this bundle from a window.load - // handler, so `load` may already have fired by the time this code runs — - // waiting for it again would skip user ID setup entirely on that path. - if (document.readyState === 'complete') { - installUserIdModules(); - } else { - window.addEventListener( - 'load', - () => { - installUserIdModules(); - }, - { once: true } - ); - } } + // TS writes no userSync values beyond managed userIds (and autoRefresh on a + // late seed), so the publisher's or Prebid's auctionDelay applies from the + // first auction. } export { pbjs }; diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json b/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json index 38061761d3..32f2db4b1d 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json +++ b/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json @@ -13,75 +13,48 @@ "modules": [ { "moduleName": "connectIdSystem", - "configNames": ["connectId"], - "eidSources": ["yahoo.com"] + "configNames": ["connectId"] }, { "moduleName": "criteoIdSystem", - "configNames": ["criteo"], - "eidSources": ["criteo.com"] + "configNames": ["criteo"] }, { "moduleName": "id5IdSystem", - "configNames": ["id5Id"], - "eidSources": ["id5-sync.com"] + "configNames": ["id5Id"] }, { "moduleName": "identityLinkIdSystem", - "configNames": ["identityLink"], - "eidSources": ["liveramp.com"] + "configNames": ["identityLink"] }, { "moduleName": "liveIntentIdSystem", "configNames": ["liveIntentId"], - "eidSources": [ - "liveintent.com", - "bidswitch.net", - "liveintent.triplelift.com", - "rubiconproject.com", - "liveintent.indexexchange.com", - "openx.net", - "pubmatic.com", - "liveintent.sovrn.com", - "liveintent.unrulymedia.com", - "media.net", - "zeta-ssp.liveintent.com", - "fpid.liveintent.com", - "sharethrough.com", - "liveintent.sonobi.com", - "liveintent.vidazoo.com" - ], "notes": "Imported through a local ESM shim because the public Prebid wrapper contains CommonJS require()." }, { "moduleName": "lockrAIMIdSystem", - "configNames": ["lockrAIMId"], - "eidSources": [] + "configNames": ["lockrAIMId"] }, { "moduleName": "pairIdSystem", - "configNames": ["pairId"], - "eidSources": ["google.com"] + "configNames": ["pairId"] }, { "moduleName": "pubProvidedIdSystem", - "configNames": ["pubProvidedId"], - "eidSources": [] + "configNames": ["pubProvidedId"] }, { "moduleName": "sharedIdSystem", - "configNames": ["sharedId", "pubCommonId"], - "eidSources": ["pubcid.org"] + "configNames": ["sharedId", "pubCommonId"] }, { "moduleName": "uid2IdSystem", - "configNames": ["uid2"], - "eidSources": ["uidapi.com"] + "configNames": ["uid2"] }, { "moduleName": "unifiedIdSystem", - "configNames": ["unifiedId"], - "eidSources": ["adserver.org"] + "configNames": ["unifiedId"] } ] } diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.ts b/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.ts index b463e22026..40da6b36aa 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.ts @@ -3,7 +3,6 @@ import registryJson from './user_id_modules.json'; export interface PrebidUserIdModuleRegistryEntry { moduleName: string; configNames: string[]; - eidSources: string[]; notes?: string; } @@ -12,54 +11,10 @@ interface PrebidUserIdModuleRegistryFile { modules: PrebidUserIdModuleRegistryEntry[]; } -export interface PrebidUserIdEidLike { - source?: unknown; - uids?: Array<{ id?: unknown; atype?: unknown; ext?: unknown }>; -} - -export interface PrebidUserIdModuleResolution { - modules: string[]; - missingSources: string[]; -} - const registry = registryJson as PrebidUserIdModuleRegistryFile; -const DEFAULT_PREBID_USER_ID_MODULES = [...registry.defaultPreset]; export const PREBID_USER_ID_MODULE_REGISTRY = [...registry.modules]; -const MODULE_SORT_ORDER = new Map( - ['userId', ...DEFAULT_PREBID_USER_ID_MODULES].map((moduleName, index) => [moduleName, index]) -); - -function stableModuleSort(a: string, b: string): number { - return ( - (MODULE_SORT_ORDER.get(a) ?? Number.MAX_SAFE_INTEGER) - - (MODULE_SORT_ORDER.get(b) ?? Number.MAX_SAFE_INTEGER) || a.localeCompare(b) - ); -} - -function normalizeSource(source: unknown): string | undefined { - if (typeof source !== 'string') { - return undefined; - } - const normalized = source.trim().toLowerCase(); - return normalized.length > 0 ? normalized : undefined; -} - -function hasLiveIntentProvider(eid: PrebidUserIdEidLike): boolean { - return Array.isArray(eid.uids) - ? eid.uids.some((uid) => { - const ext = uid.ext; - return ( - ext !== null && - typeof ext === 'object' && - !Array.isArray(ext) && - (ext as Record).provider === 'liveintent.com' - ); - }) - : false; -} - export function knownUserIdConfigNames(): string[] { return [...new Set(PREBID_USER_ID_MODULE_REGISTRY.flatMap((entry) => entry.configNames))].sort(); } @@ -100,46 +55,3 @@ export function userIdConfigNameAliases(configName: string): string[] { export function userIdSubmoduleKey(configName: string): string { return findUserIdModuleEntry(configName)?.moduleName ?? configName.toLowerCase(); } - -export function resolvePrebidUserIdModulesFromEids( - eids: PrebidUserIdEidLike[] -): PrebidUserIdModuleResolution { - const modules = new Set(); - const missingSources = new Set(); - const modulesBySource = new Map(); - - for (const entry of PREBID_USER_ID_MODULE_REGISTRY) { - for (const source of entry.eidSources) { - modulesBySource.set(source.toLowerCase(), entry.moduleName); - } - } - - for (const eid of eids) { - const source = normalizeSource(eid.source); - if (!source) { - continue; - } - - const moduleName = modulesBySource.get(source); - if (moduleName) { - modules.add(moduleName); - continue; - } - - if (hasLiveIntentProvider(eid)) { - modules.add('liveIntentIdSystem'); - continue; - } - - missingSources.add(source); - } - - if (modules.size > 0) { - modules.add('userId'); - } - - return { - modules: [...modules].sort(stableModuleSort), - missingSources: [...missingSources].sort(), - }; -} diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index e88ed520cd..ee33130af3 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -25,6 +25,36 @@ describe('request.requestAds', () => { vi.restoreAllMocks(); }); + it('sends no eids in the /auction body', async () => { + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ seatbid: [] }), + }); + const { addAdUnits } = await import('../../src/core/registry'); + const { requestAds } = await import('../../src/core/request'); + document.body.innerHTML = '
'; + addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } }); + + requestAds(); + await flushRequestAds(); + + const [, init] = (globalThis.fetch as ReturnType).mock.calls[0] as [ + string, + RequestInit, + ]; + const body = JSON.parse(String(init.body)) as Record; + expect(body.adUnits, 'the body should carry the ad unit').toEqual([ + { + code: 'slot1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + bids: [], + }, + ]); + expect(body, 'requestAds should leave EIDs to the ts-eids cookie').not.toHaveProperty('eids'); + }); + it('sends fetch and renders creatives via iframe from response', async () => { // mock fetch - returns creative HTML inline in adm field const creativeHtml = '
Test Creative
'; diff --git a/crates/trusted-server-js/lib/test/fixtures/auction-body-eids.txt b/crates/trusted-server-js/lib/test/fixtures/auction-body-eids.txt new file mode 100644 index 0000000000..cec8a54bc9 --- /dev/null +++ b/crates/trusted-server-js/lib/test/fixtures/auction-body-eids.txt @@ -0,0 +1 @@ +{"adUnits":[{"code":"div-gpt-1","mediaTypes":{"banner":{"sizes":[[300,250]]}},"bids":[{"bidder":"trustedServer","params":{}}]}],"eids":[{"source":"first.example.com","inserter":"inserter-€.example.com","matcher":"first.example.com","mm":3,"uids":[{"id":"first-uid","atype":1,"ext":{"label":"日本"}}]},{"source":"bad.example.com","uids":[{"id":"bad-\ud800-uid","atype":1}]},{"source":"last.example.com","ext":{"segments":["s1"]},"uids":[{"id":"last-uid","atype":3}]}]} \ No newline at end of file diff --git a/crates/trusted-server-js/lib/test/fixtures/eid-fixture-input.ts b/crates/trusted-server-js/lib/test/fixtures/eid-fixture-input.ts new file mode 100644 index 0000000000..6c905a568e --- /dev/null +++ b/crates/trusted-server-js/lib/test/fixtures/eid-fixture-input.ts @@ -0,0 +1,23 @@ +/** + * The fixed input behind `auction-body-eids.txt` and `ts-eids-cookie.txt` + * (spec 7.1 rule 8). The middle entry's UID `id` holds a lone UTF-16 + * surrogate, which `JSON.stringify` writes as the escape `\ud800` and the + * server drops for that entry only. The first entry has non-Latin-1 text in + * `inserter` and in a UID `ext`. The last has an EID-level `ext`, which the + * cookie omits. Key order matters: the fixtures are exact bytes. + */ +export const FIXTURE_EIDS = [ + { + source: 'first.example.com', + inserter: 'inserter-€.example.com', + matcher: 'first.example.com', + mm: 3, + uids: [{ id: 'first-uid', atype: 1, ext: { label: '日本' } }], + }, + { source: 'bad.example.com', uids: [{ id: 'bad-\ud800-uid', atype: 1 }] }, + { + source: 'last.example.com', + ext: { segments: ['s1'] }, + uids: [{ id: 'last-uid', atype: 3 }], + }, +]; diff --git a/crates/trusted-server-js/lib/test/fixtures/ts-eids-cookie.txt b/crates/trusted-server-js/lib/test/fixtures/ts-eids-cookie.txt new file mode 100644 index 0000000000..732424d1c1 --- /dev/null +++ b/crates/trusted-server-js/lib/test/fixtures/ts-eids-cookie.txt @@ -0,0 +1 @@ +W3sic291cmNlIjoiZmlyc3QuZXhhbXBsZS5jb20iLCJpbnNlcnRlciI6Imluc2VydGVyLeKCrC5leGFtcGxlLmNvbSIsIm1hdGNoZXIiOiJmaXJzdC5leGFtcGxlLmNvbSIsIm1tIjozLCJ1aWRzIjpbeyJpZCI6ImZpcnN0LXVpZCIsImF0eXBlIjoxLCJleHQiOnsibGFiZWwiOiLml6XmnKwifX1dfSx7InNvdXJjZSI6ImJhZC5leGFtcGxlLmNvbSIsInVpZHMiOlt7ImlkIjoiYmFkLVx1ZDgwMC11aWQiLCJhdHlwZSI6MX1dfSx7InNvdXJjZSI6Imxhc3QuZXhhbXBsZS5jb20iLCJ1aWRzIjpbeyJpZCI6Imxhc3QtdWlkIiwiYXR5cGUiOjN9XX1d \ No newline at end of file diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/eids.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/eids.test.ts new file mode 100644 index 0000000000..296d85e3f1 --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/prebid/eids.test.ts @@ -0,0 +1,422 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, it } from 'vitest'; + +import { + collectBidderEids, + fitEidCookie, + MAX_AUCTION_EIDS_BYTES, + MAX_EID_COOKIE_BYTES, + type EidCollectionStats, +} from '../../../src/integrations/prebid/eids'; +import { FIXTURE_EIDS } from '../../fixtures/eid-fixture-input'; + +function deepFreeze(value: T, seen = new WeakSet()): T { + if (value !== null && typeof value === 'object' && !seen.has(value)) { + seen.add(value); + for (const child of Object.values(value)) { + deepFreeze(child, seen); + } + Object.freeze(value); + } + return value; +} + +/** A bidder request shaped as Prebid passes it: frozen `ortb2`, `auctionId` hidden. */ +function bidderRequest(user: unknown): unknown { + return deepFreeze({ + bidderCode: 'trustedServer', + auctionId: null, + ortb2: { user }, + }); +} + +function freshStats(): EidCollectionStats { + return { forwarded: 0, unserializable: 0, overBudget: 0 }; +} + +function utf8Length(text: string): number { + return new TextEncoder().encode(text).length; +} + +/** An entry whose `JSON.stringify` is exactly `bytes` UTF-8 bytes long. */ +function entryOfBytes(source: string, bytes: number, filler = 'x'): unknown { + const base = JSON.stringify({ source, uids: [{ id: '' }] }); + const fillerBytes = utf8Length(filler); + return { + source, + uids: [{ id: filler.repeat((bytes - utf8Length(base)) / fillerBytes) }], + }; +} + +function decodeCookie(value: string | undefined): unknown { + expect(value, 'should produce a cookie value').toBeDefined(); + return JSON.parse( + new TextDecoder().decode(Uint8Array.from(atob(value!), (char) => char.charCodeAt(0))) + ); +} + +function readFixture(name: string): string { + return fs + .readFileSync( + path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../fixtures', name), + 'utf8' + ) + .trimEnd(); +} + +const EXT_EID = { + source: 'ext.example.com', + uids: [{ id: 'ext-uid', atype: 1 }], +}; +const USER_EID = { + source: 'user.example.com', + uids: [{ id: 'user-uid', atype: 1 }], +}; + +describe('collectBidderEids', () => { + it('reads ortb2.user.ext.eids', () => { + expect(collectBidderEids(bidderRequest({ ext: { eids: [EXT_EID] } }))).toEqual([EXT_EID]); + }); + + it('reads ortb2.user.eids, where RTD modules can write after normalization', () => { + expect(collectBidderEids(bidderRequest({ eids: [USER_EID] }))).toEqual([USER_EID]); + }); + + it('reads user.ext.eids before user.eids', () => { + const request = bidderRequest({ + ext: { eids: [EXT_EID] }, + eids: [USER_EID], + }); + expect( + Object.isFrozen((request as { ortb2: { user: object } }).ortb2.user), + 'the input should be frozen as Prebid passes it' + ).toBe(true); + + expect(collectBidderEids(request)).toEqual([EXT_EID, USER_EID]); + }); + + it.each([ + ['no user', bidderRequest(undefined)], + ['an empty user', bidderRequest({})], + ['non-array paths', bidderRequest({ ext: { eids: 'x' }, eids: { source: 'x' } })], + ['no ortb2', deepFreeze({ bidderCode: 'trustedServer' })], + ])('returns undefined for %s', (_label, request) => { + expect(collectBidderEids(request)).toBeUndefined(); + }); + + it.each([undefined, null, 'request', 7])( + 'returns undefined for a bidder request that is not an object: %s', + (request) => { + expect(collectBidderEids(request)).toBeUndefined(); + } + ); + + it('returns plain-data copies without undefined fields', () => { + const openPair = { + source: 'openpair.example.com', + inserter: undefined, + matcher: undefined, + mm: 3, + uids: [{ id: 'pair-uid', atype: 571187 }], + }; + + const result = collectBidderEids(bidderRequest({ ext: { eids: [openPair] } })); + + expect(result, 'undefined provenance fields should disappear').toStrictEqual([ + { + source: 'openpair.example.com', + mm: 3, + uids: [{ id: 'pair-uid', atype: 571187 }], + }, + ]); + expect(result![0], 'the result should be a copy, not Prebid’s object').not.toBe(openPair); + }); + + it('drops an entry whose serialization throws and keeps the others', () => { + const circular: Record = {}; + circular.self = circular; + const stats = freshStats(); + + const result = collectBidderEids( + bidderRequest({ + ext: { + eids: [ + EXT_EID, + { + source: 'circular.example.com', + ext: circular, + uids: [{ id: 'c' }], + }, + { + source: 'bigint.example.com', + uids: [{ id: 'b', atype: BigInt(10) }], + }, + USER_EID, + ], + }, + }), + stats + ); + + expect(result).toEqual([EXT_EID, USER_EID]); + expect(stats).toEqual({ forwarded: 2, unserializable: 2, overBudget: 0 }); + }); + + it('keeps a set that fits the UTF-8 budget exactly', () => { + const first = entryOfBytes('first.example.com', MAX_AUCTION_EIDS_BYTES - 100); + const second = entryOfBytes('second.example.com', 100); + const stats = freshStats(); + + expect(collectBidderEids(bidderRequest({ ext: { eids: [first, second] } }), stats)).toEqual([ + first, + second, + ]); + expect(stats).toEqual({ forwarded: 2, unserializable: 0, overBudget: 0 }); + }); + + it('drops the entry that crosses the budget and every entry after it', () => { + const first = entryOfBytes('first.example.com', MAX_AUCTION_EIDS_BYTES - 100); + const second = entryOfBytes('second.example.com', 101); + const third = entryOfBytes('third.example.com', 60); + const stats = freshStats(); + + expect( + collectBidderEids(bidderRequest({ ext: { eids: [first, second, third] } }), stats) + ).toEqual([first]); + expect(stats).toEqual({ forwarded: 1, unserializable: 0, overBudget: 2 }); + }); + + it('measures the budget in UTF-8 bytes, not JavaScript string length', () => { + const ascii = entryOfBytes('ascii.example.com', 40_000); + // `€` is one UTF-16 code unit and three UTF-8 bytes: 53 bytes of JSON + // around 10,000 of them. + const multibyte = entryOfBytes('multibyte.example.com', 30_053, '€'); + expect( + JSON.stringify(ascii).length + JSON.stringify(multibyte).length, + 'precondition: the set fits by JavaScript length' + ).toBeLessThan(MAX_AUCTION_EIDS_BYTES); + + expect(collectBidderEids(bidderRequest({ ext: { eids: [ascii, multibyte] } }))).toEqual([ + ascii, + ]); + }); + + it('keeps order and repeated sources', () => { + const a1 = { source: 'a.example.com', mm: 1, uids: [{ id: 'u' }] }; + const b = { source: 'b.example.com', uids: [{ id: 'u' }] }; + const a2 = { source: 'a.example.com', mm: 2, uids: [{ id: 'u' }] }; + + expect(collectBidderEids(bidderRequest({ ext: { eids: [a1, b, a2] } }))).toEqual([a1, b, a2]); + }); + + it('forwards a lone surrogate unchanged for the server to drop', () => { + const result = collectBidderEids(bidderRequest({ ext: { eids: FIXTURE_EIDS } })); + + expect(result).toEqual(FIXTURE_EIDS); + expect(JSON.stringify(result), 'JSON.stringify should write the escape').toContain( + '"bad-\\ud800-uid"' + ); + }); + + it('passes the Prebid module shapes from spec 3.5 through as plain data', () => { + const fiftyOneDegrees = [ + { source: '51d.es', mm: 1, uids: [{ id: 'device-1', atype: 1 }] }, + { source: '51d.es', mm: 2, uids: [{ id: 'device-2', atype: 1 }] }, + ]; + const openPair = { + source: 'openpair.example.com', + inserter: undefined, + matcher: undefined, + uids: [{ id: 'p' }], + }; + const liveIntent = { + source: 'liveintent.com', + ext: { segments: ['s1', 's2'] }, + uids: [{ id: 'li', atype: 3 }], + }; + const oneKey = { + source: 'paf.example.com', + ext: { preferences: { use_browsing_for_personalization: true } }, + uids: [{ id: 'ok', atype: 1 }], + }; + const pair = { + source: 'google.com', + uids: [{ id: 'pair-id', atype: 571187 }], + }; + + const result = collectBidderEids( + bidderRequest({ + ext: { eids: [openPair, liveIntent, oneKey, pair] }, + eids: fiftyOneDegrees, + }) + ); + + expect(result).toStrictEqual([ + { source: 'openpair.example.com', uids: [{ id: 'p' }] }, + liveIntent, + oneKey, + pair, + ...fiftyOneDegrees, + ]); + }); +}); + +describe('fitEidCookie', () => { + it('removes EID-level ext and drops the duplicates that creates, comparing with sorted keys', () => { + const withExt = { + source: 'a.example.com', + ext: { segments: ['s1'] }, + uids: [{ id: 'a' }], + }; + const reordered = { uids: [{ id: 'a' }], source: 'a.example.com' }; + const other = { + source: 'b.example.com', + uids: [{ id: 'b', ext: { kept: true } }], + }; + + const fitted = fitEidCookie([withExt, reordered, other]); + + expect(decodeCookie(fitted.value)).toEqual([ + { source: 'a.example.com', uids: [{ id: 'a' }] }, + other, + ]); + expect(fitted.droppedSources).toEqual([]); + expect(fitted.trimmedSources).toEqual([]); + }); + + it('does not modify the entries it is given', () => { + const entry = { + source: 'a.example.com', + ext: { segments: ['s1'] }, + uids: [{ id: 'a' }, { id: 'b' }], + }; + const snapshot = structuredClone(entry); + + fitEidCookie([entry]); + + expect(entry).toEqual(snapshot); + }); + + it('drops whole tail sources until the cookie fits', () => { + const long = 'x'.repeat(400); + const entries = [0, 1, 2, 3, 4, 5, 6].map((index) => ({ + source: `s${index}.example.com`, + uids: [{ id: long, atype: 1 }], + })); + + const fitted = fitEidCookie(entries); + + expect(fitted.value!.length).toBeLessThanOrEqual(MAX_EID_COOKIE_BYTES); + expect(decodeCookie(fitted.value)).toEqual(entries.slice(0, 5)); + expect(fitted.droppedSources).toEqual(['s6.example.com', 's5.example.com']); + expect(fitted.trimmedSources).toEqual([]); + }); + + it('trims UIDs from the last source before dropping it', () => { + const long = 'x'.repeat(400); + const uids = Array.from({ length: 10 }, (_, index) => ({ + id: `${long}${index}`, + atype: 1, + })); + + const fitted = fitEidCookie([{ source: 'ids.example.com', uids }]); + + expect(decodeCookie(fitted.value)).toEqual([ + { source: 'ids.example.com', uids: uids.slice(0, 5) }, + ]); + expect(fitted.droppedSources).toEqual([]); + expect(fitted.trimmedSources).toEqual(['ids.example.com']); + }); + + it('reports a source trimmed and then dropped only as dropped', () => { + const huge = 'y'.repeat(2500); + const kept = { + source: 'kept.example.com', + uids: [{ id: 'kept', atype: 1 }], + }; + + const fitted = fitEidCookie([ + kept, + { + source: 'dropped.example.com', + uids: Array.from({ length: 3 }, (_, index) => ({ + id: `${huge}${index}`, + atype: 1, + })), + }, + ]); + + expect(decodeCookie(fitted.value)).toEqual([kept]); + expect(fitted.droppedSources).toEqual(['dropped.example.com']); + expect(fitted.trimmedSources).toEqual([]); + }); + + it.each([ + ['a string', 'x'.repeat(4000), '(malformed entry)'], + [ + 'an object without a uids array', + { source: 'no-uids.example.com', blob: 'x'.repeat(4000) }, + 'no-uids.example.com', + ], + ])('removes a last entry that is %s whole', (_label, last, droppedLabel) => { + const small = { source: 'small.example.com', uids: [{ id: 's' }] }; + + const fitted = fitEidCookie([small, last]); + + expect(decodeCookie(fitted.value)).toEqual([small]); + expect(fitted.droppedSources).toEqual([droppedLabel]); + }); + + it('removes a null last entry whole before trimming the entry before it', () => { + const big = { + source: 'big.example.com', + uids: [{ id: 'y'.repeat(1500) }, { id: 'z'.repeat(1500) }], + }; + + const fitted = fitEidCookie([big, null]); + + expect(decodeCookie(fitted.value)).toEqual([ + { source: 'big.example.com', uids: [big.uids[0]] }, + ]); + expect(fitted.droppedSources).toEqual(['(malformed entry)']); + expect(fitted.trimmedSources).toEqual(['big.example.com']); + }); + + it('returns no value when nothing fits', () => { + const fitted = fitEidCookie([{ source: 'huge.example.com', uids: [{ id: 'h'.repeat(5000) }] }]); + + expect(fitted).toEqual({ + value: undefined, + droppedSources: ['huge.example.com'], + trimmedSources: [], + }); + }); + + it('encodes non-Latin-1 text as UTF-8 base64 that round-trips', () => { + const entries = [ + { + source: 'ids.example.com', + inserter: 'inserter-€', + uids: [{ id: '日本' }], + }, + ]; + expect(() => btoa(JSON.stringify(entries)), 'precondition: btoa alone throws here').toThrow(); + + expect(decodeCookie(fitEidCookie(entries).value)).toEqual(entries); + }); + + it('matches btoa output for ASCII payloads', () => { + const entries = [{ source: 'ids.example.com', uids: [{ id: 'ascii-uid', atype: 1 }] }]; + + expect(fitEidCookie(entries).value).toBe(btoa(JSON.stringify(entries))); + }); + + it('produces exactly the ts-eids-cookie fixture for the fixture input', () => { + const collected = collectBidderEids(bidderRequest({ ext: { eids: FIXTURE_EIDS } })); + + expect(fitEidCookie(collected!).value).toBe(readFixture('ts-eids-cookie.txt')); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 0c209b89a1..ef67d948de 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -1,4 +1,7 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, it, expect, vi, beforeAll, beforeEach, afterAll, afterEach } from 'vitest'; function apsRenderer() { const bid = envelope.seatbid[0].bid[0]; @@ -236,6 +239,8 @@ interface TestAdapterSpec { // the external bundle in production), so tests install the mock there instead // of mocking module imports. const { + CANARY_EID, + auctionBodyLog, mockSetConfig, mockMergeConfig, mockProcessQueue, @@ -254,11 +259,39 @@ const { const mockMergeConfig = vi.fn((config: unknown) => mockSetConfig(config)); const mockProcessQueue = vi.fn(); const mockRequestBids = vi.fn(); - const mockRegisterBidAdapter = vi.fn(); + const mockRegisterBidAdapter = vi.fn( + ( + _bidder: unknown, + code: string, + spec: { buildRequests?: (...args: unknown[]) => { data?: unknown } } + ) => { + if (code !== 'trustedServer' || typeof spec?.buildRequests !== 'function') return; + const buildRequests = spec.buildRequests; + spec.buildRequests = function (this: unknown, ...args: unknown[]) { + const request = buildRequests.apply(this, args); + auctionBodyLog.bodies.push(String(request?.data)); + auctionBodyLog.total += 1; + return request; + }; + } + ); const mockMarkWinningBidAsUsed = vi.fn(); const mockOnEvent = vi.fn(); + // D1 canary (spec 7.1 rule 3): an ID only pbjs.getUserIdsAsEids() knows. It + // must never reach an /auction body or ts-eids. + const CANARY_EID = { + source: 'canary.example.com', + uids: [{ id: 'canary-uid', atype: 1 }], + }; + // Every /auction body the trustedServer adapter builds, recorded at the + // adapter so no test can bypass the canary check. + const auctionBodyLog = { bodies: [] as string[], total: 0 }; const mockGetUserIdsAsEids = vi.fn( - () => [] as Array<{ source: string; uids?: Array<{ id: string; atype?: number }> }> + () => + [CANARY_EID] as Array<{ + source: string; + uids?: Array<{ id: string; atype?: number }>; + }> ); const mockGetConfig = vi.fn(); @@ -326,6 +359,8 @@ const { }; return { + CANARY_EID, + auctionBodyLog, mockSetConfig, mockMergeConfig, mockProcessQueue, @@ -364,6 +399,7 @@ import type { TsjsApi } from '../../../src/core/types'; import { GptDiagnosticsObserver } from '../../../src/integrations/gpt_diagnostics/observer'; import { GptDiagnosticsStore } from '../../../src/integrations/gpt_diagnostics/store'; import envelope from '../../fixtures/aps-renderer-v1.json'; +import { FIXTURE_EIDS } from '../../fixtures/eid-fixture-input'; // installPrebidNpm is a per-page no-op once the sentinel is set (the module // self-init above already set it), so every test starts from a clean page. @@ -375,6 +411,109 @@ beforeEach(() => { delete mockPbjs['__tsManagedUserIdsSetConfigInstalled']; }); +function readTsEidsCookie(): string | undefined { + return document.cookie.match(/(?:^|; )ts-eids=([^;]*)/)?.[1] || undefined; +} + +function decodeBase64Utf8(value: string): string { + return new TextDecoder().decode(Uint8Array.from(atob(value), (char) => char.charCodeAt(0))); +} + +function decodedTsEidsCookie(): unknown { + const value = readTsEidsCookie(); + return value === undefined ? undefined : JSON.parse(decodeBase64Utf8(value)); +} + +function presetTsEidsCookie(value: string): void { + document.cookie = `ts-eids=${value}; Path=/; Secure; SameSite=Lax; Max-Age=60`; + expect(readTsEidsCookie(), 'the preset ts-eids cookie should read back').toBe(value); +} + +beforeAll(() => { + // jsdom silently drops a Secure cookie on a non-HTTPS URL other than + // localhost (spec 7.1 rule 5). Prove the shim's attributes survive here. + document.cookie = 'ts-eids-probe=1; Path=/; Secure; SameSite=Lax; Max-Age=60'; + expect(document.cookie, 'shim cookie attributes should be readable here').toContain( + 'ts-eids-probe=1' + ); + document.cookie = 'ts-eids-probe=; Path=/; Max-Age=0'; +}); + +afterEach(() => { + // Drain first so one leaking test cannot cascade into the next. + const bodies = auctionBodyLog.bodies.splice(0); + expect(mockGetUserIdsAsEids(), 'a test switched the getUserIdsAsEids canary off').toEqual([ + CANARY_EID, + ]); + for (const body of bodies) { + expect(body, 'the getUserIdsAsEids canary leaked into an /auction body').not.toContain( + CANARY_EID.source + ); + } + const cookie = readTsEidsCookie(); + if (cookie !== undefined) { + let readable = cookie; + try { + readable = decodeBase64Utf8(cookie); + } catch { + // A preset non-base64 test value; check it as written. + } + expect(readable, 'the getUserIdsAsEids canary leaked into ts-eids').not.toContain( + CANARY_EID.source + ); + } +}); + +afterAll(() => { + expect( + auctionBodyLog.total, + 'the canary should have inspected at least one /auction body' + ).toBeGreaterThan(0); +}); + +function deepFreeze(value: T, seen = new WeakSet()): T { + if (value !== null && typeof value === 'object' && !seen.has(value)) { + seen.add(value); + for (const child of Object.values(value)) { + deepFreeze(child, seen); + } + Object.freeze(value); + } + return value; +} + +/** + * A bidder request shaped as Prebid passes it to buildRequests: a frozen + * `ortb2` and `auctionId: null`, as Prebid's proxy hides it by default. + */ +function prebidBidderRequest(user: Record): Record { + return deepFreeze({ + bidderCode: 'trustedServer', + auctionId: null, + ortb2: { user }, + }); +} + +function trustedServerBidRequests(): Array> { + return [ + { + adUnitCode: 'div-gpt-1', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + params: {}, + }, + ]; +} + +function readFixture(name: string): string { + return fs + .readFileSync( + path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../fixtures', name), + 'utf8' + ) + .trimEnd(); +} + describe('prebid/collectBidders', () => { it('returns empty array for empty ad units', () => { expect(collectBidders([])).toEqual([]); @@ -578,7 +717,7 @@ describe('prebid/installPrebidNpm', () => { mockPbjs.que = []; mockPbjs.getConfig = mockGetConfig; mockGetUserIdsAsEids.mockReset(); - mockGetUserIdsAsEids.mockReturnValue([]); + mockGetUserIdsAsEids.mockReturnValue([CANARY_EID]); mockGetConfig.mockReset(); document.cookie = 'ts-eids=; Path=/; Max-Age=0'; testWindow.__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; @@ -1320,7 +1459,14 @@ describe('prebid/installPrebidNpm', () => { 'preserves publisher userSync and autoRefresh=%s when seeding late IDs', (autoRefresh) => { testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; - const userSync = { userIds: [], auctionDelay: 300, syncEnabled: false, autoRefresh }; + const userSync = { + userIds: [], + auctionDelay: 300, + syncDelay: 50, + filterSettings: { image: { bidders: '*', filter: 'include' } }, + syncEnabled: false, + autoRefresh, + }; mockGetConfig.mockImplementation((key?: string) => { if (key === 'userSync.userIds') return []; if (key === 'userSync') return userSync; @@ -2416,203 +2562,303 @@ describe('prebid/installPrebidNpm', () => { expect(payload.eids).toBeUndefined(); }); - it('buildRequests includes current Prebid EIDs in the /auction payload', () => { + it('forwards an opaque LiveRamp envelope from the bidder request unchanged', () => { const spec = getAdapterSpec(); - mockGetUserIdsAsEids.mockReturnValue([ - { - source: 'id5-sync.com', - uids: [{ id: 'ID5_abc', atype: 1 }], - }, - { - source: 'sharedid.org', - uids: [{ id: 'shared_123' }, { id: 'shared_456', atype: 3 }], - }, - { - source: 'google.com', - uids: [{ id: 'pair_123', atype: 571187 }], - }, - ]); + const liveramp = { + source: 'liveramp.com', + uids: [{ id: 'opaque-test-envelope', atype: 3 }], + }; - const result = spec.buildRequests([ - { - adUnitCode: 'div-gpt-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - params: {}, - }, - ]); + const request = spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [liveramp] } }) + ); - const payload = JSON.parse(result.data); - expect(payload.eids).toEqual([ - { - source: 'id5-sync.com', - uids: [{ id: 'ID5_abc', atype: 1 }], - }, - { - source: 'sharedid.org', - uids: [{ id: 'shared_123' }, { id: 'shared_456', atype: 3 }], - }, - { - source: 'google.com', - uids: [{ id: 'pair_123', atype: 571187 }], - }, - ]); + expect(JSON.parse(String(request.data)).eids).toEqual([liveramp]); }); - it('forwards the opaque LiveRamp envelope as a liveramp.com EID', () => { + it('forwards bidderRequest.ortb2 EIDs with EID-level fields, user.ext.eids first', () => { const spec = getAdapterSpec(); - mockGetUserIdsAsEids.mockReturnValue([ - { - source: 'liveramp.com', - uids: [{ id: 'opaque-test-envelope', atype: 3 }], - }, - ]); + const extEid = { + source: 'adserver.org', + inserter: 'adserver.org', + matcher: 'adserver.org', + mm: 4, + uids: [{ id: 'fictional-tdid', atype: 1, ext: { rtiPartner: 'TDID' } }], + }; + const userEid = { + source: '51d.es', + mm: 1, + uids: [{ id: 'fictional-device', atype: 1 }], + }; - const request = spec.buildRequests([ - { - adUnitCode: 'div-gpt-1', - bidId: 'bid-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - params: {}, - }, - ]); + const result = spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [extEid] }, eids: [userEid] }) + ); - expect(JSON.parse(request.data).eids).toEqual([ - { - source: 'liveramp.com', - uids: [{ id: 'opaque-test-envelope', atype: 3 }], - }, + expect(JSON.parse(String(result.data)).eids, 'should forward both paths as sent').toEqual([ + extEid, + userEid, ]); + expect(mockGetUserIdsAsEids, 'should never read getUserIdsAsEids').not.toHaveBeenCalled(); + expect( + mockGetConfig.mock.calls.some(([key]) => key === 'ortb2'), + 'should never read the global ortb2 config' + ).toBe(false); }); - it('drops empty and malformed LiveRamp envelope values', () => { + it('sends exactly the auction-body fixture for the fixture input', () => { const spec = getAdapterSpec(); - mockGetUserIdsAsEids.mockReturnValue([ - { - source: 'liveramp.com', - uids: [ - { id: '' }, - { id: undefined as unknown as string }, - { id: 'opaque-valid-envelope', atype: 3 }, - ], - }, - { source: '', uids: [{ id: 'opaque-invalid-source' }] }, - ]); - const request = spec.buildRequests([ - { - adUnitCode: 'div-gpt-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - params: {}, - }, - ]); + const result = spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: FIXTURE_EIDS } }) + ); - expect(JSON.parse(request.data).eids).toEqual([ - { - source: 'liveramp.com', - uids: [{ id: 'opaque-valid-envelope', atype: 3 }], - }, - ]); + expect(String(result.data), 'TSJS should produce the Rust fixture byte for byte').toBe( + readFixture('auction-body-eids.txt') + ); }); - it('never writes an opaque LiveRamp envelope to logs', () => { - const sentinel = 'opaque-envelope-must-not-be-logged'; - const spies = [ - vi.spyOn(log, 'debug').mockImplementation(() => {}), - vi.spyOn(log, 'info').mockImplementation(() => {}), - vi.spyOn(log, 'warn').mockImplementation(() => {}), - vi.spyOn(log, 'error').mockImplementation(() => {}), - ]; + it('writes exactly the ts-eids fixture for the fixture input', () => { const spec = getAdapterSpec(); - mockGetUserIdsAsEids.mockReturnValue([ - { source: 'liveramp.com', uids: [{ id: sentinel, atype: 3 }] }, - ]); - spec.buildRequests([ - { - adUnitCode: 'div-gpt-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - params: {}, - }, - ]); + spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: FIXTURE_EIDS } }) + ); - const logged = spies.flatMap((spy) => spy.mock.calls).flat(); - expect(logged.some((value) => JSON.stringify(value).includes(sentinel))).toBe(false); + expect(readTsEidsCookie()).toBe(readFixture('ts-eids-cookie.txt')); }); - it('buildRequests clears stale ts-eids cookie when current Prebid EIDs are absent', () => { + it('writes ts-eids from the body set without EID-level ext', () => { const spec = getAdapterSpec(); - document.cookie = 'ts-eids=stale-value'; - mockGetUserIdsAsEids.mockReturnValue([]); + const withExt = { + source: 'liveintent.example.com', + ext: { segments: ['s1'] }, + uids: [{ id: 'li-uid', atype: 3 }], + }; + const plain = { + source: 'ids.example.com', + uids: [{ id: 'ids-uid', atype: 1 }], + }; - spec.buildRequests([ - { - adUnitCode: 'div-gpt-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - params: {}, - }, + const result = spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [withExt, plain] } }) + ); + + expect(JSON.parse(String(result.data)).eids, 'the body keeps EID-level ext').toEqual([ + withExt, + plain, ]); + expect(decodedTsEidsCookie(), 'the cookie copies the body set without EID-level ext').toEqual( + [{ source: 'liveintent.example.com', uids: [{ id: 'li-uid', atype: 3 }] }, plain] + ); + }); + + it('clears ts-eids before the request when the bidder request holds no EIDs', () => { + const spec = getAdapterSpec(); + presetTsEidsCookie('preset-value'); - expect(document.cookie).toBe(''); + const result = spec.buildRequests(trustedServerBidRequests(), prebidBidderRequest({})); + + const payload = JSON.parse(String(result.data)); + expect(payload.adUnits[0].code, 'the request should still be built').toBe('div-gpt-1'); + expect(payload.eids, 'the body should carry no eids').toBeUndefined(); + expect(readTsEidsCookie(), 'the cookie should be cleared').toBeUndefined(); }); - it('buildRequests preserves uid ext and sanitizes invalid atype values', () => { + it('clears ts-eids when no EID fits the cookie budget but still sends the body', () => { + const debugSpy = vi.spyOn(log, 'debug').mockImplementation(() => {}); const spec = getAdapterSpec(); - mockGetUserIdsAsEids.mockReturnValue([ - { - source: 'adserver.org', - uids: [ - { - id: 'uid-with-ext', - atype: 1, - ext: { provider: 'liveintent.com', rtiPartner: 'TDID' }, - }, - { - id: 'uid-bad-atype', - atype: 2_147_483_648, - ext: { keep: true }, - }, - { - id: 'uid-float-atype', - atype: 1.5, - }, - ], - }, - ]); + presetTsEidsCookie('preset-value'); + const entry = { source: 'huge.example.com', uids: [{ id: 'h'.repeat(5000) }] }; - const result = spec.buildRequests([ - { - adUnitCode: 'div-gpt-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - params: {}, - }, - ]); + const result = spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [entry] } }) + ); - const payload = JSON.parse(result.data); - expect(payload.eids).toEqual([ - { - source: 'adserver.org', - uids: [ - { - id: 'uid-with-ext', - atype: 1, - ext: { provider: 'liveintent.com', rtiPartner: 'TDID' }, - }, - { - id: 'uid-bad-atype', - ext: { keep: true }, - }, - { - id: 'uid-float-atype', + expect(JSON.parse(String(result.data)).eids, 'the body keeps the entry').toEqual([entry]); + expect(readTsEidsCookie(), 'the cookie should be cleared').toBeUndefined(); + expect(debugSpy).toHaveBeenCalledWith('[tsjs-prebid] forwarding bidder EIDs', { + forwarded: 1, + unserializable: 0, + overBudget: 0, + cookie: 'cleared', + }); + }); + + describe('when document.cookie writes throw', () => { + function withBlockedCookieWrites(run: () => void): void { + const descriptor = Object.getOwnPropertyDescriptor(Document.prototype, 'cookie')!; + Object.defineProperty(document, 'cookie', { + configurable: true, + get: () => descriptor.get!.call(document), + set: () => { + throw new Error('cookies blocked'); + }, + }); + try { + run(); + } finally { + delete (document as unknown as Record).cookie; + } + } + + it('still builds the request with the entry when the cookie write fails', () => { + vi.spyOn(log, 'warn').mockImplementation(() => {}); + const spec = getAdapterSpec(); + const entry = { source: 'liveramp.com', uids: [{ id: 'lr-uid', atype: 3 }] }; + + withBlockedCookieWrites(() => { + const result = spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [entry] } }) + ); + + const payload = JSON.parse(String(result.data)); + expect(payload.adUnits[0].code, 'the request should still be built').toBe('div-gpt-1'); + expect(payload.eids, 'the body should carry the entry').toEqual([entry]); + }); + }); + + it('still builds the request without eids when the cookie clear fails', () => { + vi.spyOn(log, 'warn').mockImplementation(() => {}); + const spec = getAdapterSpec(); + + withBlockedCookieWrites(() => { + const result = spec.buildRequests(trustedServerBidRequests(), prebidBidderRequest({})); + + const payload = JSON.parse(String(result.data)); + expect(payload.adUnits[0].code, 'the request should still be built').toBe('div-gpt-1'); + expect(payload.eids, 'the body should carry no eids').toBeUndefined(); + }); + }); + }); + + it('sends no eids and leaves ts-eids alone without a bidder request object', () => { + const debugSpy = vi.spyOn(log, 'debug').mockImplementation(() => {}); + const spec = getAdapterSpec(); + presetTsEidsCookie('preset-value'); + + const result = spec.buildRequests(trustedServerBidRequests()); + + expect(JSON.parse(String(result.data)).adUnits[0].code).toBe('div-gpt-1'); + expect(JSON.parse(String(result.data)).eids).toBeUndefined(); + expect(readTsEidsCookie(), 'the cookie should be unchanged').toBe('preset-value'); + expect( + debugSpy.mock.calls.some( + ([message]) => typeof message === 'string' && message.includes('no bidder request') + ), + 'should explain at debug level' + ).toBe(true); + }); + + it('lets the later of two overlapping requests own ts-eids', () => { + const spec = getAdapterSpec(); + const first = { source: 'first.example.com', uids: [{ id: 'first-uid' }] }; + const second = { source: 'second.example.com', uids: [{ id: 'second-uid' }] }; + + spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [first] } }) + ); + spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [second] } }) + ); + + expect(decodedTsEidsCookie()).toEqual([second]); + }); + + it('forwards ortb2 EIDs and writes ts-eids without the User ID module', () => { + const spec = getAdapterSpec(); + const original = mockPbjs.getUserIdsAsEids; + mockPbjs.getUserIdsAsEids = undefined as unknown as typeof mockGetUserIdsAsEids; + try { + const lockrId5 = { + source: 'id5-sync.com', + inserter: 'inserter.example.com', + mm: 2, + uids: [{ id: 'fictional-id5', atype: 1 }], + }; + + const result = spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [lockrId5] } }) + ); + + expect(JSON.parse(String(result.data)).eids).toEqual([lockrId5]); + expect(decodedTsEidsCookie()).toEqual([lockrId5]); + } finally { + mockPbjs.getUserIdsAsEids = original; + } + }); + + it('logs EID counts and the cookie decision at debug level, never values', () => { + log.setLevel('debug'); + const spies = [ + vi.spyOn(log, 'debug').mockImplementation(() => {}), + vi.spyOn(log, 'info').mockImplementation(() => {}), + vi.spyOn(log, 'warn').mockImplementation(() => {}), + vi.spyOn(log, 'error').mockImplementation(() => {}), + ]; + try { + const spec = getAdapterSpec(); + + spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ + ext: { + eids: [{ source: 'ids.example.com', uids: [{ id: 'secret-uid-value' }] }], }, - ], - }, - ]); + }) + ); + + expect(spies[0]).toHaveBeenCalledWith('[tsjs-prebid] forwarding bidder EIDs', { + forwarded: 1, + unserializable: 0, + overBudget: 0, + cookie: 'written', + }); + const logged = spies.flatMap((spy) => spy.mock.calls).flat(); + expect( + logged.some((value) => String(JSON.stringify(value)).includes('secret-uid-value')), + 'no log argument may contain a UID value' + ).toBe(false); + } finally { + log.setLevel('warn'); + } + }); + + it('keeps the trim log when the cookie had to drop sources', () => { + const debugSpy = vi.spyOn(log, 'debug').mockImplementation(() => {}); + const spec = getAdapterSpec(); + const long = 'x'.repeat(400); + const entries = [0, 1, 2, 3, 4, 5, 6].map((index) => ({ + source: `s${index}.example.com`, + uids: [{ id: long, atype: 1 }], + })); + + const result = spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: entries } }) + ); + + expect(JSON.parse(String(result.data)).eids, 'the body keeps every entry').toEqual(entries); + expect(decodedTsEidsCookie(), 'the cookie keeps what fits').toEqual(entries.slice(0, 5)); + expect(debugSpy).toHaveBeenCalledWith( + '[tsjs-prebid] ts-eids cookie exceeded 3072 bytes; dropped sources: s6.example.com, s5.example.com' + ); + expect(debugSpy).toHaveBeenCalledWith('[tsjs-prebid] forwarding bidder EIDs', { + forwarded: 7, + unserializable: 0, + overBudget: 0, + cookie: 'trimmed', + }); }); it('buildRequests uses custom endpoint when configured', () => { @@ -2739,6 +2985,66 @@ describe('prebid/installPrebidNpm', () => { }); }); + it('exposes the getUserIdsAsEids canary on the pbjs the shim uses', () => { + const pbjs = installPrebidNpm() as unknown as { + getUserIdsAsEids?: () => unknown; + }; + + expect(pbjs.getUserIdsAsEids).toBe(mockGetUserIdsAsEids); + expect(pbjs.getUserIdsAsEids?.()).toEqual([CANARY_EID]); + }); + + function tcfEnforcementHandler(): (event: unknown) => void { + installPrebidNpm(); + const registration = mockOnEvent.mock.calls.find(([event]) => event === 'tcf2Enforcement'); + expect(registration, 'the shim should listen for tcf2Enforcement').toBeDefined(); + return registration![1] as (event: unknown) => void; + } + + it.each([ + ['biddersBlocked', { biddersBlocked: ['trustedServer'], eidsBlocked: [] }], + ['eidsBlocked', { biddersBlocked: [], eidsBlocked: ['trustedServer'] }], + ])('clears ts-eids when tcf2Enforcement lists trustedServer in %s', (_list, blocked) => { + const handler = tcfEnforcementHandler(); + presetTsEidsCookie('preset-value'); + + handler({ + storageBlocked: [], + analyticsBlocked: [], + ufpdBlocked: [], + geoBlocked: [], + ...blocked, + }); + + expect(readTsEidsCookie()).toBeUndefined(); + }); + + it('leaves ts-eids alone when tcf2Enforcement lists only other bidders', () => { + const handler = tcfEnforcementHandler(); + presetTsEidsCookie('preset-value'); + + handler({ biddersBlocked: ['appnexus'], eidsBlocked: ['appnexus'] }); + + expect(readTsEidsCookie()).toBe('preset-value'); + }); + + it('leaves ts-eids alone when an auction builds no trustedServer request', () => { + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + presetTsEidsCookie('preset-value'); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], + } as unknown as RequestBidsArg); + + expect(mockRequestBids, 'the auction should have run').toHaveBeenCalled(); + expect(readTsEidsCookie(), 'only buildRequests may write or clear the cookie').toBe( + 'preset-value' + ); + }); + describe('requestBids shim', () => { beforeEach(() => { testWindow.__tsjs_prebid = { @@ -3156,202 +3462,6 @@ describe('prebid/installPrebidNpm', () => { ); expect(hasTsBidder).toBe(true); }); - - it('syncs a structured ts-eids cookie after bidsBackHandler', () => { - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); - mockGetUserIdsAsEids.mockReturnValue([ - { - source: 'sharedid.org', - uids: [ - { id: 'shared_123', atype: 3 }, - { id: 'shared_456', ext: { provider: 'example' } }, - ], - }, - ]); - - const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], - } as unknown as RequestBidsArg); - - const cookieValue = document.cookie.match(/(?:^|; )ts-eids=([^;]+)/)?.[1]; - expect(cookieValue).toBeDefined(); - expect(JSON.parse(atob(cookieValue!))).toEqual([ - { - source: 'sharedid.org', - uids: [ - { id: 'shared_123', atype: 3 }, - { id: 'shared_456', ext: { provider: 'example' } }, - ], - }, - ]); - }); - - it('preserves an opaque LiveRamp envelope in the ts-eids cookie', () => { - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); - mockGetUserIdsAsEids.mockReturnValue([ - { - source: 'liveramp.com', - uids: [{ id: 'opaque-test-envelope', atype: 3 }], - }, - ]); - - const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], - } as unknown as RequestBidsArg); - - const cookieValue = document.cookie.match(/(?:^|; )ts-eids=([^;]+)/)?.[1]; - expect(cookieValue).toBeDefined(); - expect(JSON.parse(atob(cookieValue!))).toEqual([ - { - source: 'liveramp.com', - uids: [{ id: 'opaque-test-envelope', atype: 3 }], - }, - ]); - }); - - it('trims an oversized ts-eids cookie deterministically and logs which sources were dropped', () => { - const debugSpy = vi.spyOn(log, 'debug').mockImplementation(() => {}); - const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => {}); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); - // A long uid value per source makes the encoded payload exceed - // MAX_EID_COOKIE_BYTES (3072) once several sources are present, the - // same way a real page with many ID modules would. - const longUid = 'x'.repeat(400); - mockGetUserIdsAsEids.mockReturnValue([ - { source: 'id5-sync.com', uids: [{ id: longUid, atype: 1 }] }, - { source: 'liveramp.com', uids: [{ id: longUid, atype: 3 }] }, - { source: 'criteo.com', uids: [{ id: longUid, atype: 1 }] }, - { source: 'uidapi.com', uids: [{ id: longUid, atype: 3 }] }, - { source: 'pubcid.org', uids: [{ id: longUid, atype: 1 }] }, - { source: 'sharedid.org', uids: [{ id: longUid, atype: 1 }] }, - { source: 'dropped.example', uids: [{ id: longUid, atype: 1 }] }, - ]); - - const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], - } as unknown as RequestBidsArg); - - const cookieValue = document.cookie.match(/(?:^|; )ts-eids=([^;]+)/)?.[1]; - expect(cookieValue).toBeDefined(); - expect(cookieValue!.length).toBeLessThanOrEqual(3072); - - const persistedSources = (JSON.parse(atob(cookieValue!)) as Array<{ source: string }>).map( - (eid) => eid.source - ); - expect(persistedSources).not.toContain('dropped.example'); - expect(persistedSources).toContain('id5-sync.com'); - - const loggedMessage = debugSpy.mock.calls - .map((call) => call.at(-1)) - .find( - (arg): arg is string => typeof arg === 'string' && arg.includes('ts-eids cookie exceeded') - ); - expect(loggedMessage).toBeDefined(); - expect(loggedMessage).toContain('dropped.example'); - expect(warnSpy).not.toHaveBeenCalledWith(expect.stringContaining('ts-eids cookie exceeded')); - }); - - it('logs a source whose UIDs were partially trimmed but retained', () => { - const debugSpy = vi.spyOn(log, 'debug').mockImplementation(() => {}); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); - // A single source with many UIDs is trimmed down to fit rather than - // dropped outright — it stays in the payload, but with fewer UIDs. - const longUid = 'x'.repeat(400); - mockGetUserIdsAsEids.mockReturnValue([ - { - source: 'id5-sync.com', - uids: Array.from({ length: 10 }, (_, i) => ({ id: `${longUid}${i}`, atype: 1 })), - }, - ]); - - const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], - } as unknown as RequestBidsArg); - - const cookieValue = document.cookie.match(/(?:^|; )ts-eids=([^;]+)/)?.[1]; - expect(cookieValue).toBeDefined(); - - const persisted = JSON.parse(atob(cookieValue!)) as Array<{ - source: string; - uids: unknown[]; - }>; - expect(persisted.map((eid) => eid.source)).toContain('id5-sync.com'); - const persistedSource = persisted.find((eid) => eid.source === 'id5-sync.com'); - expect(persistedSource!.uids.length).toBeLessThan(10); - - const loggedMessage = debugSpy.mock.calls - .map((call) => call.at(-1)) - .find( - (arg): arg is string => typeof arg === 'string' && arg.includes('ts-eids cookie exceeded') - ); - expect(loggedMessage).toBeDefined(); - expect(loggedMessage).toContain('trimmed uids from sources: id5-sync.com'); - }); - - it('reports a source trimmed and then dropped only as dropped', () => { - const debugSpy = vi.spyOn(log, 'debug').mockImplementation(() => {}); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); - // The tail source has several UIDs that are each too large to fit: the - // trimmer first removes UIDs from it, then drops it entirely. - const hugeUid = 'y'.repeat(2500); - mockGetUserIdsAsEids.mockReturnValue([ - { source: 'id5-sync.com', uids: [{ id: 'kept', atype: 1 }] }, - { - source: 'dropped.example', - uids: Array.from({ length: 3 }, (_, i) => ({ id: `${hugeUid}${i}`, atype: 1 })), - }, - ]); - - const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], - } as unknown as RequestBidsArg); - - const cookieValue = document.cookie.match(/(?:^|; )ts-eids=([^;]+)/)?.[1]; - expect(cookieValue).toBeDefined(); - const persistedSources = (JSON.parse(atob(cookieValue!)) as Array<{ source: string }>).map( - (eid) => eid.source - ); - expect(persistedSources).toEqual(['id5-sync.com']); - - const loggedMessage = debugSpy.mock.calls - .map((call) => call.at(-1)) - .find( - (arg): arg is string => typeof arg === 'string' && arg.includes('ts-eids cookie exceeded') - ); - expect(loggedMessage).toBeDefined(); - expect(loggedMessage).toContain('dropped sources: dropped.example'); - expect(loggedMessage).not.toContain('trimmed uids from sources'); - }); - - it('clears ts-eids cookie after bidsBackHandler when no current EIDs remain', () => { - document.cookie = `ts-eids=${btoa(JSON.stringify([{ source: 'sharedid.org', uids: [{ id: 'stale' }] }]))}`; - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); - mockGetUserIdsAsEids.mockReturnValue([]); - - const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], - } as unknown as RequestBidsArg); - - expect(document.cookie).toBe(''); - }); }); }); @@ -3361,7 +3471,7 @@ describe('prebid/installPrebidNpm with server-injected config', () => { mockPbjs.requestBids = mockRequestBids; mockPbjs.adUnits = []; mockGetUserIdsAsEids.mockReset(); - mockGetUserIdsAsEids.mockReturnValue([]); + mockGetUserIdsAsEids.mockReturnValue([CANARY_EID]); document.cookie = 'ts-eids=; Path=/; Max-Age=0'; delete testWindow.__tsjs_prebid; }); @@ -4463,7 +4573,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { delete (mockPbjs as unknown as Record).__tsRemoveAdUnitWrapped; mockPbjs.adUnits = []; mockGetUserIdsAsEids.mockReset(); - mockGetUserIdsAsEids.mockReturnValue([]); + mockGetUserIdsAsEids.mockReturnValue([CANARY_EID]); // By default the manifest declares all adapters compiled in. (window as unknown as { __tsjs_prebid_bundle?: unknown }).__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; @@ -7550,7 +7660,7 @@ describe('prebid/client-side bidders', () => { mockPbjs.requestBids = mockRequestBids; mockPbjs.adUnits = []; mockGetUserIdsAsEids.mockReset(); - mockGetUserIdsAsEids.mockReturnValue([]); + mockGetUserIdsAsEids.mockReturnValue([CANARY_EID]); // By default the manifest declares all adapters compiled in. testWindow.__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; delete testWindow.__tsjs_prebid; @@ -8046,10 +8156,11 @@ describe('prebid/self-init without the external bundle', () => { }); }); -describe('prebid self-init user ID module timing', () => { - const userSyncCallCount = () => - mockSetConfig.mock.calls.filter(([arg]) => arg && typeof arg === 'object' && 'userSync' in arg) - .length; +describe('prebid self-init writes no Prebid userSync settings', () => { + const userSyncCalls = () => + mockSetConfig.mock.calls.filter( + ([arg]) => arg !== null && typeof arg === 'object' && 'userSync' in arg + ); const setReadyState = (value: DocumentReadyState) => { Object.defineProperty(document, 'readyState', { value, configurable: true }); @@ -8064,28 +8175,26 @@ describe('prebid self-init user ID module timing', () => { setReadyState('complete'); }); - it('installs user ID modules immediately when the bundle loads after window load', async () => { - // The GPT slim loader appends this bundle from a window.load handler, so - // the document is already complete — a load listener would never fire. + it('writes no userSync when the bundle loads after window load', async () => { setReadyState('complete'); await import('../../../src/integrations/prebid/index'); - expect(userSyncCallCount()).toBeGreaterThan(0); + expect(mockSetConfig, 'the shim should still apply its own config').toHaveBeenCalledWith({ + debug: false, + }); + expect(userSyncCalls(), 'TS should leave userSync to the publisher and Prebid').toEqual([]); }); - it('defers user ID modules to window load when the document is still loading', async () => { + it('writes no userSync when window load fires after install', async () => { setReadyState('loading'); await import('../../../src/integrations/prebid/index'); - - expect(userSyncCallCount()).toBe(0); - window.dispatchEvent(new Event('load')); - expect(userSyncCallCount()).toBe(1); - // { once: true } — a second load event must not reinstall. - window.dispatchEvent(new Event('load')); - expect(userSyncCallCount()).toBe(1); + expect(mockSetConfig, 'the shim should still apply its own config').toHaveBeenCalledWith({ + debug: false, + }); + expect(userSyncCalls(), 'TS should leave userSync to the publisher and Prebid').toEqual([]); }); }); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts index 06c1c33d87..3467f2b200 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts @@ -2,128 +2,21 @@ import { describe, expect, it } from 'vitest'; import { knownUserIdConfigNames, - resolvePrebidUserIdModulesFromEids, userIdSubmoduleKey, } from '../../../src/integrations/prebid/user_id_modules'; import registry from '../../../src/integrations/prebid/user_id_modules.json'; -const sampleEids = [ - { source: 'yahoo.com', uids: [{ id: 'connect-id', atype: 3 }] }, - { source: 'criteo.com', uids: [{ id: 'criteo-id', atype: 1 }] }, - { source: 'liveintent.com', uids: [{ id: 'liveintent-id', atype: 3 }] }, - { - source: 'bidswitch.net', - uids: [{ id: 'bidswitch-id', atype: 3, ext: { provider: 'liveintent.com' } }], - }, - { - source: 'liveintent.triplelift.com', - uids: [{ id: 'triplelift-id', atype: 3, ext: { provider: 'liveintent.com' } }], - }, - { - source: 'rubiconproject.com', - uids: [{ id: 'rubicon-id', atype: 3, ext: { provider: 'liveintent.com' } }], - }, - { - source: 'liveintent.indexexchange.com', - uids: [{ id: 'index-id', atype: 3, ext: { provider: 'liveintent.com' } }], - }, - { - source: 'openx.net', - uids: [{ id: 'openx-id', atype: 3, ext: { provider: 'liveintent.com' } }], - }, - { - source: 'pubmatic.com', - uids: [{ id: 'pubmatic-id', atype: 3, ext: { provider: 'liveintent.com' } }], - }, - { - source: 'liveintent.sovrn.com', - uids: [{ id: 'sovrn-id', atype: 3, ext: { provider: 'liveintent.com' } }], - }, - { - source: 'liveintent.unrulymedia.com', - uids: [{ id: 'unruly-id', atype: 3, ext: { provider: 'liveintent.com' } }], - }, - { source: 'pubcid.org', uids: [{ id: 'pubcid-id', atype: 1 }] }, - { - source: 'adserver.org', - uids: [{ id: 'tdid', atype: 1, ext: { rtiPartner: 'TDID' } }], - inserter: 'adserver.org', - matcher: 'adserver.org', - mm: 4, - }, -]; - describe('prebid user ID module registry', () => { - it('resolves the target-site EID sample to deterministic Prebid user ID modules', () => { - const result = resolvePrebidUserIdModulesFromEids(sampleEids); - - expect(result).toEqual({ - modules: [ - 'userId', - 'connectIdSystem', - 'criteoIdSystem', - 'liveIntentIdSystem', - 'sharedIdSystem', - 'unifiedIdSystem', - ], - missingSources: [], - }); - }); - it('exposes config names for modules that do not map EID sources', () => { expect(knownUserIdConfigNames()).toEqual( expect.arrayContaining(['lockrAIMId', 'pubProvidedId']) ); }); - it('maps the Google PAIR EID source to pairIdSystem', () => { - const result = resolvePrebidUserIdModulesFromEids([ - { source: 'google.com', uids: [{ id: 'pair-id' }] }, - ]); - - expect(result).toEqual({ - modules: ['userId', 'pairIdSystem'], - missingSources: [], - }); - }); - - it('maps LiveRamp EIDs to identityLinkIdSystem', () => { - expect( - resolvePrebidUserIdModulesFromEids([ - { source: 'liveramp.com', uids: [{ id: 'opaque-envelope', atype: 3 }] }, - ]) - ).toEqual({ - modules: ['userId', 'identityLinkIdSystem'], - missingSources: [], - }); - }); - it('includes identityLinkIdSystem in the checked-in default preset', () => { expect(registry.defaultPreset).toContain('identityLinkIdSystem'); }); - it('maps unknown LiveIntent provider-backed sources to liveIntentIdSystem', () => { - const result = resolvePrebidUserIdModulesFromEids([ - { - source: 'new-liveintent-partner.example', - uids: [{ ext: { provider: 'liveintent.com' } }], - }, - ]); - - expect(result).toEqual({ - modules: ['userId', 'liveIntentIdSystem'], - missingSources: [], - }); - }); - - it('reports unknown non-LiveIntent sources', () => { - const result = resolvePrebidUserIdModulesFromEids([ - { source: 'unknown.example', uids: [{ id: 'abc' }] }, - ]); - - expect(result).toEqual({ modules: [], missingSources: ['unknown.example'] }); - }); - it('keys every alias of one module on the same submodule', () => { expect(userIdSubmoduleKey('pubCommonId')).toBe(userIdSubmoduleKey('sharedId')); expect(userIdSubmoduleKey('SHAREDID')).toBe(userIdSubmoduleKey('sharedId')); @@ -136,6 +29,14 @@ describe('prebid user ID module registry', () => { expect(userIdSubmoduleKey('NotARegisteredModule')).toBe('notaregisteredmodule'); }); + it('carries no EID source metadata', () => { + expect(registry.modules.length, 'the registry should still list modules').toBeGreaterThan(0); + expect( + registry.modules.filter((entry) => 'eidSources' in entry).map((entry) => entry.moduleName), + 'no entry should keep eidSources' + ).toEqual([]); + }); + it('keeps submodule keys and config names unique across registry entries', () => { // Actual registrations are checked by prebid-user-id-registry.test.mjs; // these checks prevent separate entries from sharing an ownership key. diff --git a/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs b/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs index 24db647d31..8d152d6ec5 100644 --- a/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs +++ b/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs @@ -307,10 +307,13 @@ describe('tsjs-prebid production artifacts', () => { expect(analyticsArtifact.bundleCode.length).toBeGreaterThan(200_000); // A value-import of Prebid or a private rendering helper would multiply // the shim size. The bound sits just above the normal compact shim output, - // which is roughly 40 KB: tight enough that material growth has to be + // which is roughly 42 KB: it carries the bidder-request EID collector, the + // ts-eids cookie fitting and encoding, and the tcf2Enforcement listener, + // which replaced the getUserIdsAsEids collector and the bidsBackHandler + // cookie sync. The bound is tight enough that material growth has to be // noticed and re-justified here, and far enough below a multiplication // that one still fails loudly. The bundle beside it is 200 KB and up. - expect(shimCode.length).toBeLessThan(41_000); + expect(shimCode.length).toBeLessThan(42_500); expect(shimCode).toContain('markWinningBidAsUsed'); }); diff --git a/crates/trusted-server-js/lib/test/prebid-eid-fidelity.test.mjs b/crates/trusted-server-js/lib/test/prebid-eid-fidelity.test.mjs new file mode 100644 index 0000000000..616ef85394 --- /dev/null +++ b/crates/trusted-server-js/lib/test/prebid-eid-fidelity.test.mjs @@ -0,0 +1,485 @@ +// @vitest-environment node + +// Real Prebid.js arms for the trustedServer EID contract (spec 7.5). Unit +// tests mock window.pbjs; this suite evaluates the generated external bundle +// and the served shim together in a JSDOM page and captures /auction bodies. +// Every arm except the D4 control fails against a shim built from main. + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { JSDOM, requestInterceptor } from 'jsdom'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +import { moduleBuildOptions } from '../build-module-options.mjs'; +import { main } from '../build-prebid-external.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const libDir = path.resolve(__dirname, '..'); + +const PAGE_ORIGIN = 'https://pub.example.com'; +const HOLD_LOAD_PATH = '/hold-window-load.css'; +const UNIFIED_ID_PATH = '/unified-id'; +const UNIFIED_ID_DELAY_MS = 200; +const IDENTITY_LINK_GVL_ID = 97; + +const CONTROL_EID = { + source: 'control.example.com', + uids: [{ id: 'control-uid', atype: 1 }], +}; +const PRESET_COOKIE_EIDS = [{ source: 'preset.example.com', uids: [{ id: 'preset-uid' }] }]; +const LIVERAMP_EID = { + source: 'liveramp.com', + uids: [{ id: 'fictional-idl-envelope', atype: 3 }], +}; +const UNIFIED_ID_EID = { + source: 'adserver.org', + inserter: 'adserver.org', + matcher: 'adserver.org', + mm: 4, + uids: [{ id: 'fictional-tdid', atype: 1, ext: { rtiPartner: 'TDID' } }], +}; +// The only safe form of a trustedServer TCF exception (spec 3.7). +const FULL_BASIC_ADS_RULE = { + purpose: 'basicAds', + enforcePurpose: true, + enforceVendor: true, + softVendorExceptions: ['trustedServer'], +}; + +let outputDirectory; +let bundleCode; +let shimCode; + +beforeAll(async () => { + outputDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'trusted-server-prebid-eids-')); + await main([ + '--modules-json', + JSON.stringify({ + bidder: ['appnexusBidAdapter'], + userId: ['sharedIdSystem', 'unifiedIdSystem', 'identityLinkIdSystem'], + }), + '--out', + outputDirectory, + ]); + const manifest = JSON.parse(fs.readFileSync(path.join(outputDirectory, 'manifest.json'), 'utf8')); + bundleCode = fs.readFileSync(path.join(outputDirectory, manifest.filename), 'utf8'); + + const { build } = await import('vite'); + await build( + moduleBuildOptions({ + name: 'prebid', + entryPath: path.join(libDir, 'src', 'integrations', 'prebid', 'index.ts'), + outDir: outputDirectory, + }) + ); + shimCode = fs.readFileSync(path.join(outputDirectory, 'tsjs-prebid.js'), 'utf8'); +}, 240_000); + +afterAll(() => { + fs.rmSync(outputDirectory, { recursive: true, force: true }); +}); + +function tcfData(vendorConsents) { + return { + gdprApplies: true, + tcString: 'CPexampleTCStringForTests', + eventStatus: 'tcloaded', + cmpStatus: 'loaded', + apiVersion: '2', + purpose: { + consents: { 1: true, 2: true, 3: true, 4: true }, + legitimateInterests: {}, + }, + vendor: { consents: vendorConsents, legitimateInterests: {} }, + publisher: { restrictions: {} }, + specialFeatureOptins: {}, + }; +} + +/** A CMP that answers at once: Purposes 1 to 4 consented, the given vendors only. */ +function tcfApi(vendorConsents) { + let nextListenerId = 1; + return (command, _version, callback) => { + if (command === 'addEventListener') { + callback({ ...tcfData(vendorConsents), listenerId: nextListenerId++ }, true); + } else if (command === 'getTCData') { + callback(tcfData(vendorConsents), true); + } else if (command === 'removeEventListener') { + callback(true, true); + } + }; +} + +/** + * A publisher page whose window.load is held by a stylesheet until + * releaseLoad(). Every fetch goes to a spy, installed before the bundle is + * evaluated, that records /auction bodies and answers the unifiedId endpoint + * after UNIFIED_ID_DELAY_MS. + */ +function createPage({ cookies = [], vendorConsents } = {}) { + let releaseHeldLoad; + const loadHeld = new Promise((resolve) => { + releaseHeldLoad = resolve; + }); + const dom = new JSDOM( + ``, + { + url: `${PAGE_ORIGIN}/article`, + runScripts: 'outside-only', + pretendToBeVisual: true, + resources: { + interceptors: [ + requestInterceptor(async (request) => { + if (new URL(request.url).pathname === HOLD_LOAD_PATH) { + await loadHeld; + return new Response('', { + status: 200, + headers: { 'Content-Type': 'text/css' }, + }); + } + return new Response('', { status: 204 }); + }), + ], + }, + } + ); + const pageWindow = dom.window; + + // jsdom drops Secure cookies on non-HTTPS origins (spec 7.1 rule 5). + pageWindow.document.cookie = 'ts-eids-probe=1; Path=/; Secure; SameSite=Lax'; + expect(pageWindow.document.cookie, 'Secure cookies should work on the page origin').toContain( + 'ts-eids-probe=1' + ); + pageWindow.document.cookie = 'ts-eids-probe=; Path=/; Max-Age=0'; + for (const cookie of cookies) { + pageWindow.document.cookie = cookie; + } + + const auctionBodies = []; + pageWindow.fetch = vi.fn(async (resource, init) => { + const url = new URL(typeof resource === 'string' ? resource : resource.url, PAGE_ORIGIN); + if (url.pathname === '/auction') { + const body = + init?.body ?? (typeof resource === 'object' ? await resource.clone().text() : undefined); + auctionBodies.push(JSON.parse(body)); + } + if (url.pathname === UNIFIED_ID_PATH) { + await new Promise((resolve) => setTimeout(resolve, UNIFIED_ID_DELAY_MS)); + return new Response(JSON.stringify({ TDID: 'fictional-tdid' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response('{}', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }); + pageWindow.Request = class PageRequest extends Request { + constructor(resource, init) { + super(typeof resource === 'string' ? new URL(resource, PAGE_ORIGIN).href : resource, init); + } + }; + pageWindow.Headers = Headers; + pageWindow.Response = Response; + pageWindow.AbortController = AbortController; + if (!('isSecureContext' in pageWindow)) { + pageWindow.isSecureContext = true; + } + if (vendorConsents) { + pageWindow.__tcfapi = tcfApi(vendorConsents); + } + + // Mirror the server's head-injected state. + pageWindow.eval('window.pbjs = { que: [], cmd: [] };'); + pageWindow.__tsjs_prebid = { clientSideBidders: [], serverSideBidders: [] }; + + return { + pageWindow, + auctionBodies, + install() { + pageWindow.eval(bundleCode); + pageWindow.eval(shimCode); + }, + async releaseLoad() { + releaseHeldLoad(); + await vi.waitFor(() => expect(pageWindow.document.readyState).toBe('complete')); + }, + async close() { + releaseHeldLoad(); + // Let in-flight User ID work settle before jsdom tears down. + await new Promise((resolve) => setTimeout(resolve, UNIFIED_ID_DELAY_MS + 100)); + pageWindow.close(); + }, + }; +} + +function readTsEidsCookie(pageWindow) { + const value = pageWindow.document.cookie.match(/(?:^|; )ts-eids=([^;]*)/)?.[1]; + if (!value) { + return undefined; + } + return JSON.parse( + new TextDecoder().decode(Uint8Array.from(atob(value), (char) => char.charCodeAt(0))) + ); +} + +function presetTsEidsCookie() { + return `ts-eids=${btoa(JSON.stringify(PRESET_COOKIE_EIDS))}; Path=/; Secure; SameSite=Lax`; +} + +function requestBids(pageWindow, adUnits) { + return new Promise((resolve) => { + pageWindow.pbjs.requestBids({ + adUnits, + timeout: 1000, + bidsBackHandler: () => resolve(), + }); + }); +} + +/** An ad unit with no publisher bidders: the shim adds only trustedServer. */ +function trustedServerAdUnits() { + return [ + { + code: 'ad-slot-1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + bids: [], + }, + ]; +} + +/** An ad unit with a client-side bidder that has a GVL ID (appnexus, 32). */ +function clientBidderAdUnits() { + return [ + { + code: 'ad-slot-1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + bids: [{ bidder: 'appnexus', params: { placementId: 1 } }], + }, + ]; +} + +function tcfEnforcementResults(pageWindow) { + return pageWindow.pbjs + .getEvents() + .filter((event) => event.eventType === 'tcf2Enforcement') + .map((event) => event.args); +} + +describe('trustedServer EIDs with real Prebid.js', () => { + it('D1: never forwards an ID restricted to another bidder', async () => { + const page = createPage({ + cookies: ['_sharedid=fictional-shared-id; Path=/'], + }); + try { + page.install(); + expect(page.pageWindow.document.readyState, 'install runs before load').toBe('loading'); + page.pageWindow.pbjs.setConfig({ + userSync: { + auctionDelay: 300, + userIds: [ + { + name: 'sharedId', + bidders: ['appnexus'], + storage: { type: 'cookie', name: '_sharedid', expires: 30 }, + }, + ], + }, + ortb2: { user: { ext: { eids: [CONTROL_EID] } } }, + }); + + await requestBids(page.pageWindow, trustedServerAdUnits()); + expect(page.pageWindow.document.readyState, 'the auction runs before load').toBe('loading'); + + expect( + page.pageWindow.pbjs.getUserIdsAsEids().map((eid) => eid.source), + 'precondition: the restricted module produced its ID' + ).toContain('pubcid.org'); + expect( + page.auctionBodies.map((body) => body.eids), + 'the /auction body should carry the control and not the restricted ID' + ).toEqual([[CONTROL_EID]]); + expect(readTsEidsCookie(page.pageWindow), 'ts-eids should copy the body set').toEqual([ + CONTROL_EID, + ]); + } finally { + await page.close(); + } + }, 60_000); + + it('D3: forwards same-source entries separately with their own EID-level fields', async () => { + const first = { + source: 'ids.example.com', + inserter: 'inserter-a.example.com', + mm: 2, + ext: { segments: ['s1'] }, + uids: [{ id: 'ids-uid', atype: 1 }], + }; + const second = { + source: 'ids.example.com', + inserter: 'inserter-b.example.com', + matcher: 'matcher-b.example.com', + mm: 3, + uids: [{ id: 'ids-uid', atype: 1 }], + }; + const page = createPage(); + try { + page.install(); + expect(page.pageWindow.document.readyState, 'install runs before load').toBe('loading'); + page.pageWindow.pbjs.setConfig({ + ortb2: { user: { ext: { eids: [first, second] } } }, + }); + + await requestBids(page.pageWindow, trustedServerAdUnits()); + expect(page.pageWindow.document.readyState, 'the auction runs before load').toBe('loading'); + + expect(page.auctionBodies.map((body) => body.eids)).toEqual([[first, second]]); + const { ext: _eidExt, ...firstWithoutExt } = first; + expect( + readTsEidsCookie(page.pageWindow), + 'ts-eids should hold both entries without EID-level ext' + ).toEqual([firstWithoutExt, second]); + } finally { + await page.close(); + } + }, 60_000); + + it('D4: the first auction after window.load waits for a pending User ID module', async () => { + const page = createPage(); + try { + page.install(); + expect(page.pageWindow.document.readyState, 'step 1 starts before load').toBe('loading'); + await page.releaseLoad(); + + page.pageWindow.pbjs.setConfig({ + userSync: { + userIds: [ + { + name: 'unifiedId', + params: { url: `${PAGE_ORIGIN}${UNIFIED_ID_PATH}` }, + }, + ], + }, + }); + expect(page.pageWindow.document.readyState, 'step 2 runs after load').toBe('complete'); + expect( + page.pageWindow.pbjs.getConfig('userSync').auctionDelay, + "Prebid's default auctionDelay should apply" + ).toBe(500); + + await requestBids(page.pageWindow, trustedServerAdUnits()); + + expect( + page.auctionBodies.map((body) => body.eids), + 'the first /auction should carry the delayed unifiedId' + ).toEqual([[UNIFIED_ID_EID]]); + } finally { + await page.close(); + } + }, 60_000); + + it('D4 control: the first auction before window.load waits, on every build', async () => { + const page = createPage(); + try { + page.install(); + page.pageWindow.pbjs.setConfig({ + userSync: { + userIds: [ + { + name: 'unifiedId', + params: { url: `${PAGE_ORIGIN}${UNIFIED_ID_PATH}` }, + }, + ], + }, + }); + expect(page.pageWindow.document.readyState, 'the auction runs before load').toBe('loading'); + + await requestBids(page.pageWindow, trustedServerAdUnits()); + + // A projection, not the full entry: main drops EID-level fields, and + // this control must pass on main and on the branch alike. + expect( + page.auctionBodies.map((body) => + (body.eids ?? []).map((eid) => [eid.source, eid.uids.map((uid) => uid.id)]) + ), + 'the spy and the module should deliver unifiedId to the first auction' + ).toEqual([[['adserver.org', ['fictional-tdid']]]]); + } finally { + await page.close(); + } + }, 60_000); + + async function runTcfPage({ withRule }) { + const page = createPage({ + cookies: ['idl_env=fictional-idl-envelope; Path=/', presetTsEidsCookie()], + vendorConsents: { [IDENTITY_LINK_GVL_ID]: true }, + }); + page.install(); + expect(page.pageWindow.document.readyState, 'install runs before load').toBe('loading'); + expect(readTsEidsCookie(page.pageWindow), 'precondition: ts-eids is preset').toEqual( + PRESET_COOKIE_EIDS + ); + page.pageWindow.pbjs.setConfig({ + consentManagement: { + gdpr: { + cmpApi: 'iab', + timeout: 500, + ...(withRule ? { rules: [FULL_BASIC_ADS_RULE] } : {}), + }, + }, + userSync: { + auctionDelay: 300, + userIds: [ + { + name: 'identityLink', + params: { pid: '999', notUse3P: true }, + storage: { type: 'cookie', name: 'idl_env', expires: 15 }, + }, + ], + }, + ortb2: { user: { ext: { eids: [CONTROL_EID] } } }, + }); + await requestBids(page.pageWindow, clientBidderAdUnits()); + expect(page.pageWindow.document.readyState, 'the auction runs before load').toBe('loading'); + return page; + } + + it('TCF: without an exception no client EIDs reach TS and ts-eids is cleared', async () => { + const page = await runTcfPage({ withRule: false }); + try { + expect( + tcfEnforcementResults(page.pageWindow).some((result) => + result.biddersBlocked.includes('trustedServer') + ), + 'positive control: Prebid blocked trustedServer' + ).toBe(true); + expect(page.auctionBodies, 'no /auction request should be sent').toEqual([]); + expect(readTsEidsCookie(page.pageWindow), 'ts-eids should be cleared').toBeUndefined(); + } finally { + await page.close(); + } + }, 60_000); + + it('TCF: with the full basicAds rule TS gets its EIDs and other bidders stay blocked', async () => { + const page = await runTcfPage({ withRule: true }); + try { + // Prebid appends module EIDs after ortb2 entries (spec 3.2 step 3). + expect(page.auctionBodies.map((body) => body.eids)).toEqual([[CONTROL_EID, LIVERAMP_EID]]); + const results = tcfEnforcementResults(page.pageWindow); + expect( + results.some((result) => result.biddersBlocked.includes('appnexus')), + 'the rule must not switch off enforcement for other bidders' + ).toBe(true); + expect( + results.some((result) => result.biddersBlocked.includes('trustedServer')), + 'trustedServer should not be blocked with the rule' + ).toBe(false); + } finally { + await page.close(); + } + }, 60_000); +}); diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index f6c8f72d31..721226f38b 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -212,6 +212,49 @@ curl -X POST https://edge.example.com/auction \ -d '{"adUnits":[{"code":"banner","mediaTypes":{"banner":{"sizes":[[300,250]]}}}]}' ``` +#### Browser EIDs (`eids`) + +The optional `eids` array carries OpenRTB 2.6 EID objects from the browser: + +```json +{ + "eids": [ + { + "source": "ids.example.com", + "inserter": "inserter.example.com", + "matcher": "ids.example.com", + "mm": 4, + "ext": { "segments": ["s1"] }, + "uids": [ + { "id": "fictional-uid", "atype": 1, "ext": { "partner": "example" } } + ] + } + ] +} +``` + +Each entry is validated on its own. A malformed entry or field is dropped and +the request proceeds; only a malformed body outside `eids` rejects the request. + +| Field | Rule | +| --------------------- | -------------------------------------------------------- | +| entry | JSON object that parses on its own | +| `source` | string, not blank, at most 255 UTF-8 bytes | +| `uids` | at least one valid UID; the first 32 valid UIDs are kept | +| `uids[].id` | string, not blank, at most 512 bytes | +| `uids[].atype`, `mm` | integer from 0 to 2,147,483,647 | +| `uids[].ext`, `ext` | JSON object, at most 2,048 bytes as compact JSON | +| `inserter`, `matcher` | string, not blank, at most 255 bytes | +| array | duplicates dropped, then the first 64 entries kept | + +Entries keep their order and a source may repeat. When the body has no valid +entry, the `ts-eids` cookie is used with the same rules. Browser EIDs are used +only for a request with an allowed EC. + +The response's `x-ts-eids` header carries the consent-gated EIDs sent to +providers, base64-encoded JSON including EID-level fields, capped at 4,096 +bytes. `x-ts-eids-truncated: true` marks a truncated value. + #### PBS stored-request intent The reserved `trustedServer` bid accepts `storedRequest` inside `params`, beside @@ -901,7 +944,7 @@ curl -u 'admin:' \ ### GET /\_ts/admin/eids -Parses the request's `ts-eids` and `sharedId` cookies and previews which configured partner IDs cookie ingestion would match or drop. It performs request inspection only: it does not read or write KV and is available on every adapter. The preview lists candidate matches from the cookies alone; it does not apply the TCF Purpose 1 + 4 consent gate on identity-graph writes, does not see `/auction` request-body EIDs, and does not account for partner UIDs already stored in KV. +Parses the request's `ts-eids` and `sharedId` cookies and previews which configured partner IDs cookie ingestion would match or drop. It performs request inspection only: it does not read or write KV and is available on every adapter. The preview lists candidate matches from the cookies alone; it does not apply the TCF Purpose 1 + 4 consent gate on identity-graph writes, does not see `/auction` request-body EIDs, and does not account for partner UIDs already stored in KV. Cookie entries are validated one by one with the `/auction` body rules, so a mistyped field drops only that field, and the returned `eids` include `inserter`, `matcher` and `mm` when present. The `ingest.unmatched` diagnostics still read the raw cookie, before validation. After successful authentication this endpoint always returns `200 OK`; missing or malformed cookies are represented by `cookie_present`, `sharedid_present`, and `parse_error`. The `ingest.matched` and `ingest.unmatched` arrays show the ingestion preview. Each unmatched entry contains its `source` and either a `no_partner` reason when no configured partner recognizes it or `no_valid_uid` when the partner exists but every supplied UID is empty or exceeds the storage limit. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index f39ad638c6..40ebe24059 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -652,6 +652,11 @@ be at least 32 bytes. Keep it stable to preserve EC identifier continuity. `source_domain` is the canonical partner key. It matches incoming OpenRTB EID `source` values and is also used as the EC KV `ids` map key. ::: +`openrtb_atype` sets the `atype` of EIDs that Trusted Server builds from KV. +It applies only where KV supplies a source the browser did not send; for a +source in the browser's EIDs, the browser's own entry and `atype` are +forwarded instead. + `api_token` is optional. Set it to a key in `trusted_server_secrets` only when the partner calls the inbound identify or batch-sync APIs. A partner without `api_token` remains available for source-domain lookup, bidstream EIDs, and diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index cccb04daee..bd917df473 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -189,6 +189,13 @@ Expected decoded payload contains: - `source = ssp.example.com` - `uids[0].id = ` +When the request also carries browser EIDs, the decoded header lists them +first, exactly as the `trustedServer` adapter sent them (including `inserter`, +`matcher` and `mm`), followed by KV-only partner sources. The browser copies +the same EIDs, without EID-level `ext`, into the `ts-eids` cookie, which +navigations and `GET /_ts/page-bids` ingest. `/auction` ingests its own body +EIDs instead of the cookie. + ## 8) Fastly KV Operational Checks List stores: diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 113066cdaf..979e268b70 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -148,13 +148,21 @@ flowchart LR ### Prebid EID Flow -Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a response. It collects matched partner UIDs from three request-local sources, applied in this order so that, when two sources carry the same partner, the later source's UID is the candidate offered to KV: - -1. the `ts-eids` cookie, -2. the `eids` array in the `/auction` request body, -3. the `sharedId` cookie. - -The `/auction` body carries the full EID set Prebid.js resolved, so `/auction` ingestion is not limited by the size cap TSJS applies to the `ts-eids` cookie. The cookie still bridges Prebid user ID modules with the identity graph on requests that carry no EID body, such as `GET /_ts/page-bids` and page navigations. +Browser EIDs reach the identity graph when Trusted Server finalizes a +response. Which browser input it reads depends on the route: + +- `POST /auction` reads only the client EIDs the auction itself used: its + request body, or the `ts-eids` cookie when the body has none. An earlier + page's cookie is never ingested next to a body. +- Publisher navigations, admitted `GET /_ts/page-bids` requests and newly + generated ECs read the `ts-eids` cookie. +- Every route then applies the `sharedId` cookie last. + +The TSJS Prebid adapter writes `ts-eids` in `buildRequests`, before each +`trustedServer` request is sent: the request's EIDs without EID-level `ext`, +trimmed to 3,072 bytes, as base64 of UTF-8 JSON. It clears the cookie when that +set is empty or when Prebid's TCF rules block `trustedServer`. Auctions that do +not include `trustedServer` leave it unchanged. These Prebid EID writes follow the same consent rule as bidstream EID forwarding: when a TCF signal is present, Purpose 1 (store/access device) and Purpose 4 (personalized ads) must both be consented, and when GDPR applies without a TCF signal the writes are withheld. A user who consents to Purpose 1 but not Purpose 4 keeps their EC, but no EIDs from the request are persisted. @@ -166,12 +174,9 @@ sequenceDiagram participant TS as Trusted Server participant KV as KV Store - Prebid->>Prebid: Auction completes - Prebid->>TSJS: bidsBackHandler fires - TSJS->>Prebid: getUserIdsAsEids() - Prebid-->>TSJS: [{source, uids: [{id, atype}]}] - TSJS->>TSJS: Base64 encode full OpenRTB-style EID array
[{source, uids:[{id, atype, ext?}]}] - TSJS->>B: document.cookie = "ts-eids=..." + Prebid->>TSJS: buildRequests(bidRequests, bidderRequest) + TSJS->>TSJS: Copy bidderRequest.ortb2 EIDs
drop EID-level ext, fit 3,072 bytes + TSJS->>B: document.cookie = "ts-eids=..." (before the request is sent) Note over B,TS: Next eligible EID-sync request B->>TS: Document navigation, POST /auction,
or admitted GET /_ts/page-bids @@ -179,7 +184,7 @@ sequenceDiagram TS->>KV: Add missing partner IDs in one conditional write
skip when UIDs already match
(requires TCF Purpose 1 + 4 under GDPR) ``` -Current TSJS writers preserve the full OpenRTB-style `{source, uids:[...]}` shape in `ts-eids`. The server remains backward-compatible with earlier flattened `{source, id, atype}` cookies during rollout, but new cookies use the structured `uids[]` form. +`ts-eids` holds OpenRTB-style `{source, uids:[...]}` entries, with `inserter`, `matcher` and `mm` when Prebid set them. The server validates each entry with the same rules as the `/auction` body and drops only what is malformed. Earlier flattened `{source, id, atype}` cookies are still accepted. The `sharedId` cookie follows a similar path but is written directly by Prebid's SharedID module rather than by TSJS. The server reads it separately and maps it via the `sharedid.org` source domain. @@ -208,24 +213,24 @@ sequenceDiagram TS->>KV: Upsert ids[source_domain] = UID else Prebid sync seeds browser EIDs B->>B: Prebid User ID modules resolve IDs - B->>TSJS: getUserIdsAsEids() + B->>TSJS: buildRequests(bidRequests, bidderRequest) TSJS->>B: Write ts-eids cookie
Base64 OpenRTB-style EIDs (size-capped) B->>TS: Next eligible request with ts-eids TS->>KV: Add missing matched partner UIDs
defer different stored values end Note over B,TS: Prebid-routed auction - B->>B: getUserIdsAsEids() for current auction + B->>B: Prebid builds the trustedServer bidder request B->>TS: POST /auction
adUnits + eids[] + ts-ec cookie TS->>KV: Resolve EC-backed source-domain IDs KV-->>TS: Stored partner UIDs - TS->>TS: Convert stored UIDs to EIDs
Merge + dedupe with request eids[]
Apply consent gating + TS->>TS: Request eids[] as sent
+ KV EIDs for sources not sent
Apply consent gating TS->>PS: OpenRTB request
user.ext.eids = merged EID set PS->>DSP: OpenRTB bid request
user.ext.eids preserved for bidders DSP-->>PS: OpenRTB bid response PS-->>TS: OpenRTB seatbid response TS-->>B: Auction response + x-ts-eids header when available - TS->>KV: Add missing partner UIDs from request eids[]
+ ts-eids/sharedId cookies (TCF Purpose 1 + 4 under GDPR) + TS->>KV: Add missing partner UIDs from request eids[]
+ sharedId cookie (TCF Purpose 1 + 4 under GDPR) ``` The relevant OpenRTB structure forwarded to Prebid Server and downstream partners is: @@ -263,7 +268,7 @@ The relevant OpenRTB structure forwarded to Prebid Server and downstream partner } ``` -Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source + uid.id`. Browser-supplied EIDs, from the `/auction` request body, the `ts-eids` cookie, or the `sharedId` cookie, only fill partner UIDs that are missing from KV; a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). +Browser EIDs are forwarded as sent; KV adds EIDs only for registered sources the browser did not send. Browser-supplied EIDs, from the `/auction` request body, the `ts-eids` cookie, or the `sharedId` cookie, only fill partner UIDs that are missing from KV; a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). ### Pull-Sync Completeness Marker diff --git a/docs/guide/integrations/lockr.md b/docs/guide/integrations/lockr.md index ef9e57b0f3..b0ede7e124 100644 --- a/docs/guide/integrations/lockr.md +++ b/docs/guide/integrations/lockr.md @@ -88,28 +88,38 @@ User data is stored securely in Lockr's privacy vault: ### Extended ID (EID) Support -Lockr identities are injected into OpenRTB bid requests as Extended Identifiers: +The Lockr SDK writes its ID5 EID into Prebid's first-party data with +`pbjs.setConfig({ ortb2: { user: { ext: { eids } } } })`. Prebid includes it in +the `trustedServer` bidder request, and the adapter forwards it to `/auction` +with its EID-level fields, for example: ```json { - "user": { - "ext": { - "eids": [ - { - "source": "lockr.io", - "uids": [ - { - "id": "lockr-user-id-123", - "atype": 1 - } - ] - } - ] - } - } + "source": "id5-sync.com", + "inserter": "inserter.example.com", + "mm": 2, + "uids": [{ "id": "fictional-id5-id", "atype": 1 }] } ``` +The SDK writes the EID after its own network call, so the first auction on a +page may run before it is available. Later auctions include it, and requests +without an EID body get it from the `ts-eids` cookie and KV. + +To store the ID in the EC identity graph, register the source: + +```toml +[[ec.partners]] +name = "ID5 via Lockr" +source_domain = "id5-sync.com" +bidstream_enabled = true +``` + +Where GDPR applies and Prebid's TCF enforcement is active, Prebid sends the +`trustedServer` adapter no EIDs unless the publisher grants the exception +described in +[TCF and the `trustedServer` adapter](./prebid.md#tcf-and-the-trustedserver-adapter). + ## Use Cases ### 1. Cookieless Identity Resolution @@ -224,9 +234,11 @@ curl https://edge.example.com/ **Solutions**: -- Verify OpenRTB request builder includes `user.ext.eids` -- Check integration is registered in IntegrationRegistry -- Ensure `contribute_eids()` method is implemented +- Confirm `pbjs.getEvents()` shows a `bidRequested` event for `trustedServer` + whose `ortb2.user.ext.eids` contains the `id5-sync.com` entry +- Under GDPR, confirm the publisher's Prebid config grants `trustedServer` the + full TCF exception +- Decode the `/auction` response's `x-ts-eids` header to see what was forwarded ## Performance diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index 18db20e237..08a55eb991 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -339,8 +339,8 @@ A settled result that denies consent still seeds the managed entry; `tcfControl` then blocks the vendor call and the storage write, which is the enforcement path this integration relies on. A CMP that recovers later — the user completes its UI, or a delayed first response arrives — activates the managed entries at that -point, with `userSync.autoRefresh` briefly enabled so Prebid initializes modules -added after its first pass. +point, with `userSync.autoRefresh` enabled so Prebid initializes modules +added after its first pass. Because Prebid accumulates `userSync` settings, `autoRefresh` then stays enabled for the rest of the page. This scope is deliberately narrow. It applies only to the automatic TCF collector that Trusted Server configures for its own managed modules. A @@ -659,9 +659,26 @@ paths. ## User ID Modules -Prebid.js can expose publisher-configured User ID Module output via -`pbjs.getUserIdsAsEids()`. The TSJS Prebid shim reads those current-request -EIDs after auctions and forwards them to Trusted Server when they are available. +Prebid.js places the EIDs each bidder may receive in that bidder's +`bidderRequest.ortb2`: User ID module output, `setConfig({ortb2})` and +`setBidderConfig` data, and RTD output, with IDs restricted to other bidders +left out and Prebid's activity controls applied. The `trustedServer` adapter +forwards exactly the `user.ext.eids` and `user.eids` entries in its own bidder +request, with their `inserter`, `matcher`, `mm` and `ext`. It never reads +`pbjs.getUserIdsAsEids()`, which also returns IDs restricted to other bidders. + +### Timing and `userSync` + +Trusted Server writes no Prebid `userSync` settings other than the managed +`userIds` entries below (and `autoRefresh` when a late CMP answer seeds them). +The publisher's `auctionDelay`, `syncDelay`, `syncsPerBidder` and +`filterSettings`, or Prebid's defaults, apply from the first auction. Prebid's +default `auctionDelay` is 500 ms: an auction waits up to that long while User +ID modules are still resolving, and not at all once they have settled. Set +`auctionDelay` and `filterSettings` in your own Prebid configuration if the +defaults do not suit you. Prebid's default allows image user syncs only. +Vendors that write `ortb2` after their own network call, such as the Lockr +SDK, are not awaited; their IDs reach later requests. User ID submodule inclusion comes from `bundle.modules.user_id`. The available modules and default preset are checked in at @@ -845,16 +862,23 @@ IdentityLink resolves asynchronously. A new browser's first auction can run before RampID is available; later auctions can include it without blocking the page or auction. When available, the opaque value follows the standard path: -1. `pbjs.getUserIdsAsEids()` exposes an entry whose source is `liveramp.com`. -2. The current `/auction` request includes that entry. -3. Trusted Server merges and consent-gates it, then forwards it to Prebid - Server as `user.ext.eids`. +1. Prebid places the `liveramp.com` entry in the `trustedServer` bidder + request's `ortb2.user.ext.eids`. +2. The adapter forwards that entry in the `/auction` body and, before sending + the request, copies the body's EIDs (without EID-level `ext`) into the + bounded `ts-eids` cookie. +3. Trusted Server consent-gates it and forwards it to Prebid Server as + `user.ext.eids`. 4. After the response, the same `/auction` request ingests the body entry into an EC/KV partner configured with `source_domain = "liveramp.com"`, when that partner UID is not already stored. -5. The browser also persists the opaque value in the bounded `ts-eids` cookie, - which lets requests without an EID body, such as `GET /_ts/page-bids` and - page navigations, ingest it. +5. Requests without an EID body, such as `GET /_ts/page-bids` and page + navigations, read and ingest the `ts-eids` cookie. + +Where GDPR applies and Prebid's TCF enforcement is active, Prebid sends the +`trustedServer` adapter no bid requests unless the publisher grants it an +exception (see [TCF and the `trustedServer` adapter](#tcf-and-the-trustedserver-adapter)), +and the adapter clears `ts-eids`, so no browser EIDs reach Trusted Server. Trusted Server treats the RampID envelope as an opaque string. Do not log, decode, publish, or dimension metrics by the value. Source names, counts, @@ -936,17 +960,28 @@ not passed by this implementation. ## Identity Forwarding -Trusted Server uses a **hybrid EID forwarding model** for Prebid-routed auctions: - -1. **Current-request EIDs from Prebid.js** are read from `pbjs.getUserIdsAsEids()` in the browser and sent in the `/auction` request body. -2. **Server-side EIDs from the EC/KV identity graph** are resolved on the edge from the current EC ID. -3. Trusted Server **merges and deduplicates** both sets before calling Prebid Server. -4. The merged result is forwarded downstream as `user.ext.eids` in the OpenRTB request. -5. After the response, Trusted Server adds matched partner UIDs that are missing from the EC identity graph, reading the `ts-eids` cookie, then the `/auction` request-body EIDs, then the `sharedId` cookie. Because the body carries the untrimmed EID set, `/auction` ingestion is not limited by the size-capped `ts-eids` cookie. - -The `ts-eids` cookie still matters for requests without an EID body, such as `GET /_ts/page-bids` and page navigations, where it is the source for both EID fallback and identity-graph ingestion. Identity-graph EID writes follow the same consent rule as bidstream forwarding: under GDPR, TCF Purpose 1 and Purpose 4 must both be consented. - -This means Prebid auctions get same-request transparency for browser-resolved IDs without giving up the durability of the server-managed EC identity graph. +Trusted Server combines browser EIDs with the EC/KV identity graph: + +1. **Browser EIDs** are the entries Prebid placed in the `trustedServer` + bidder request. The adapter sends them in the `/auction` body as plain data + and copies them, without EID-level `ext`, into the `ts-eids` cookie before + the request is sent. +2. **Server-side EIDs** are resolved from the EC/KV identity graph for + partners with `bidstream_enabled = true`. +3. Browser entries are forwarded first, exactly as sent. KV adds entries only + for registered sources the browser did not send. +4. Consent gating applies to the merged set, which goes out as `user.ext.eids` + and is echoed in the `x-ts-eids` response header. +5. After the response, `/auction` adds matched partner UIDs that KV does not + yet hold from its own client set (the body, or the cookie when the body has + none), then from the `sharedId` cookie. Other routes read the `ts-eids` + cookie instead. + +The server validates every browser entry on its own, whether it came in the +body or the cookie. A malformed entry or field is dropped and the rest of the +request proceeds. Identity-graph writes follow the same consent rule as +bidstream forwarding: under GDPR, TCF Purpose 1 and Purpose 4 must both be +consented. ### Identity flow @@ -957,38 +992,86 @@ sequenceDiagram participant K as EC + KV identity graph participant P as Prebid Server - B->>B: User ID modules resolve EIDs - B->>T: POST /auction\n(adUnits + current-request eids) + B->>B: Prebid builds the trustedServer bidder request (ortb2 EIDs) + B->>B: Copy its EIDs into ts-eids (no EID-level ext) + B->>T: POST /auction\n(adUnits + the bidder request's eids) + T->>T: Validate each entry T->>K: Resolve EC-backed source-domain IDs K-->>T: KV-derived EIDs - T->>T: Merge + dedupe client + KV EIDs + T->>T: Browser entries as sent, then KV-only sources T->>T: Apply consent gating T->>P: OpenRTB request\nuser.ext.eids = merged set P-->>T: OpenRTB bid response - T-->>B: Auction response - T->>K: Ingest body eids + ts-eids/sharedId cookies\n(TCF Purpose 1 + 4 under GDPR) + T-->>B: Auction response + x-ts-eids + T->>K: Ingest this request's client EIDs + sharedId cookie\n(TCF Purpose 1 + 4 under GDPR) ``` -### Merge and deduplication rules +### Merge rules -- Client-request EIDs and KV-resolved EIDs are merged by `source` -- UIDs are deduplicated by `source + id` -- If the same UID appears in both places, it is sent only once downstream -- Distinct UIDs under the same source are preserved -- Consent gating is applied to the **merged** set before forwarding +- Browser entries are forwarded in the order Prebid sent them and are never + altered or combined. A source can appear in more than one entry, for example + `adserver.org` from two modules with different `matcher` values. +- KV adds one entry for each registered source no browser entry carries. + Sources are compared case-insensitively, ignoring a trailing dot. +- For a source the browser sends, KV's stored UID and its configured + `openrtb_atype` are not used. +- Consent gating is applied to the merged set before forwarding. ### What reaches Prebid Server The downstream Prebid Server request includes: - `user.id` when EC forwarding is allowed -- `user.ext.eids` containing the merged, deduplicated EID set +- `user.ext.eids` with the browser entries and their `inserter`, `matcher`, + `mm` and `ext`, followed by KV-only sources - forwarded browser cookies (subject to consent-forwarding mode) -In practice, this gives operators both: +Before rolling this out, send one request with a repeated source and all four +EID-level fields to each configured provider in staging and confirm it is +accepted. + +### TCF and the `trustedServer` adapter + +The `trustedServer` adapter has no GVL ID. Where GDPR applies and Prebid's TCF +enforcement is active (the publisher configures `consentManagement.gdpr`, or +Trusted Server enables it for managed User IDs), Prebid's default rules remove +`trustedServer` from the auction and withhold its EIDs. Trusted Server then +receives no browser EIDs from the adapter, and the adapter clears `ts-eids` +when Prebid reports the block. + +Granting `trustedServer` an exception is the publisher's decision. Use only +the full rule: + +```js +pbjs.setConfig({ + consentManagement: { + gdpr: { + // ...the publisher's existing CMP settings... + rules: [ + { + purpose: 'basicAds', + enforcePurpose: true, + enforceVendor: true, + softVendorExceptions: ['trustedServer'], + }, + ], + }, + }, +}) +``` + +::: warning +A publisher rule replaces Prebid's default rule for that purpose. A rule +written as `{ purpose: 'basicAds', softVendorExceptions: ['trustedServer'] }`, +without `enforcePurpose: true` and `enforceVendor: true`, switches off both +checks for **every** bidder. Do not use `vendorExceptions`, which also skips +the purpose check. +::: -- **same-request identity transparency** for Prebid User ID Module output, and -- **future-request continuity** through cookie ingestion and KV-backed partner resolution. +With the full rule, `trustedServer` still needs Purpose 2 consent and skips +only the vendor check. A publisher that sets `eidsRequireP4Consent` must also +add the same full rule for `personalizedAds`; without it `trustedServer` gets +bid requests with no EIDs and `ts-eids` is cleared after every auction. ## Endpoints @@ -1053,7 +1136,7 @@ The shared OpenRTB driver builds Prebid Server requests: - Sets `tagid` from the slot ID - Adds site metadata with publisher domain, a validated publisher-owned page URL with query and fragment removed, `site.publisher` from the domain, and the browser `Referer` as `site.ref`. Removing query and fragment data from `site.page` can reduce contextual targeting or per-page reporting for sites whose page identity depends on query parameters - Injects EC ID in the user object -- Merges current-request browser EIDs with KV-resolved EIDs and forwards the deduplicated result as `user.ext.eids` +- Forwards the browser EIDs as sent, followed by KV-resolved EIDs for registered sources the browser did not send, as `user.ext.eids` - Forwards user consent string and sets the GDPR flag based on geo and consent presence - Translates the `Sec-GPC` header to a US Privacy string (`us_privacy`) - Extracts `DNT` and `Accept-Language` headers into device fields diff --git a/docs/superpowers/implementation-notes/2026-10-18-prebid-eid-fidelity.md b/docs/superpowers/implementation-notes/2026-10-18-prebid-eid-fidelity.md new file mode 100644 index 0000000000..ce7294a855 --- /dev/null +++ b/docs/superpowers/implementation-notes/2026-10-18-prebid-eid-fidelity.md @@ -0,0 +1,235 @@ +# Prebid EID Fidelity Implementation Notes + +Red log for the tests this work adds or changes (spec 7.1 rule 1, A12). Each +entry names the test, the command, and the failure seen before the change, or, +for a regression net, the line temporarily reverted and the failure it caused. + +## Environment + +- Date: 2026-10-19 +- Branch: `feat/1246-prebid-eid-fidelity` +- Rust: rustc 1.95.0 (59807616e 2026-04-14) +- Node: v24.12.0 + +## Task 1: `Eid` carries OpenRTB 2.6 EID-level fields + +Command: + +```bash +cargo test-fastly --no-run 2>&1 | grep -E "^error" | sort | uniq -c +``` + +Failures (the new tests and helpers do not compile against the unchanged `Eid`): + +- `eid_serializes_openrtb_provenance_fields_when_present`, `eid_deserializes_provenance_fields_and_keeps_rejecting_unknown_keys`, `eid_header_carries_provenance_fields_and_repeated_sources`, `to_openrtb_forwards_eid_provenance_fields_and_repeated_sources`, `aps_request_forwards_eid_provenance_fields_and_repeated_sources`, `standard_profile_forwards_eid_provenance_fields_and_repeated_sources`: `error[E0560]: struct 'openrtb::Eid' has no field named 'inserter'` (also `matcher`, `mm`, `ext`; 4 each) +- `eid_without_provenance_fields_serializes_as_before`: `error[E0599]: no function or associated item named 'new' found for struct 'openrtb::Eid'` +- `eid_deserializes_provenance_fields_and_keeps_rejecting_unknown_keys`: `error[E0369]: binary operation '==' cannot be applied to type 'openrtb::Eid'` + +## Task 2: One parser for browser EIDs + +Command (each test run alone, because a failed assertion aborts the wasm test binary): + +```bash +cargo test-fastly -- ec::client_eids::tests:: --exact +``` + +Failures (the stub `parse_client_eids` returns no entries, so each assertion on the parsed output fails and the Viceroy run aborts on the panic): + +- `keeps_a_valid_entry_with_every_field`, `drops_only_the_entry_that_does_not_parse`, `drops_entries_that_are_not_objects`, `validates_source`, `drops_an_entry_without_a_valid_uid`, `validates_uid_ids`, `keeps_the_first_32_valid_uids`, `reads_atype_and_mm_as_integers_in_range`, `validates_ext_objects_and_their_size`, `validates_inserter_and_matcher`, `drops_duplicates_before_the_entry_cap`, `keeps_the_first_64_entries`, `keeps_order_and_repeated_sources_and_ignores_unknown_keys`, `logs_dropped_entries_and_fields_by_name_and_source_without_values`: `assert_eq!` or `assert!` on the empty stub output panics (wasm abort) +- `yields_nothing_for_a_payload_that_is_not_an_array`: passes against the stub, as expected (regression net) + +## Task 3: Body and cookie go through the shared parser + +Command (each test run alone, because a failed assertion aborts the wasm test binary): + +```bash +cargo test-fastly -- :: --exact +``` + +Failures (against the unchanged body and cookie readers): + +- `tsjs_auction_body_fixture_drops_only_the_unparsable_entry` (body rejected, provider saw `None`), `tsjs_cookie_fixture_drops_only_the_unparsable_entry` (cookie rejected, provider saw `Some(Null)`), `auction_body_eid_level_fields_reach_the_auction_request` (EID-level fields dropped): wasm abort on assertion. Moved to Task 4: they also depend on the merge rewrite; the old merge drops EID-level fields. +- `structured_cookie_keeps_eid_level_fields`, `structured_cookie_drops_a_mistyped_field_instead_of_the_whole_cookie`, `structured_cookie_isolates_an_unparsable_entry`, `structured_cookie_applies_the_entry_cap`: assertion panic (wasm abort) +- `structured_cookie_diagnostics_report_raw_uid_strings`: also FAILS on unchanged code (the strict cookie parser keeps the 513-byte UID), so it is a red-first test, not a regression net; no revert proof needed +- `auction_uses_the_cookie_when_no_body_entry_is_valid`: passes against unchanged code (regression net) +- Revert proof: with `.filter(|eids| !eids.is_empty())` removed from `resolve_client_auction_eids`, `auction_uses_the_cookie_when_no_body_entry_is_valid` FAILS (`left: Some(Null)`); line restored + +## Task 4: Forward browser entries as sent; KV fills absent sources + +Command (each test run alone, because a failed assertion aborts the wasm test binary): + +```bash +cargo test-fastly -- :: --exact +``` + +Failures (against the old merge): + +- `merge_case_1_forwards_a_browser_entry_with_its_fields`, `merge_case_2_keeps_the_browser_atype_for_a_shared_uid`, `merge_case_3_drops_the_stale_kv_uid_for_a_browser_source`, `merge_case_4_appends_kv_sources_the_browser_did_not_send`, `merge_case_5_keeps_same_source_entries_with_different_fields_apart`, `merge_case_6_matches_sources_as_the_registry_normalizes_them`, `merge_auction_eids_does_not_union_kv_uids_into_a_browser_source`, `merge_auction_eids_keeps_browser_metadata_on_conflict`, `publisher::tests::body_less_auction_eids_put_cookie_entries_first_and_kv_only_sources_after`: assertion panic (wasm abort); the old merge drops EID-level fields, puts KV first, unions UIDs and lets KV's atype and ext win +- `tsjs_auction_body_fixture_drops_only_the_unparsable_entry`, `tsjs_cookie_fixture_drops_only_the_unparsable_entry`, `auction_body_eid_level_fields_reach_the_auction_request` (deferred from Task 3): assertion panic (wasm abort); the old merge rebuilds entries without EID-level fields +- `merge_case_7_forwards_kv_entries_when_the_browser_sent_none`, `merge_drops_entries_without_uids`, `merge_auction_eids_deduplicates_client_and_resolved_ids`: pass against the old merge (regression nets) +- Revert proof: with `merged.push(eid);` replaced by `let _ = eid;`, `merge_case_7_forwards_kv_entries_when_the_browser_sent_none` FAILS (wasm abort); line restored +- Revert proof: with `.filter(|eid| !eid.uids.is_empty())` removed from the browser branch, `merge_drops_entries_without_uids` FAILS (wasm abort); line restored + +## Task 5: KV ingestion on /auction reads only the auction's own client set + +Command (each test run alone, because a failed assertion aborts the wasm test binary): + +```bash +cargo test-fastly -- :: --exact +``` + +Failures (against the old ingestion, which always read the cookie first): + +- `auction::endpoints::tests::auction_ingests_its_body_eids_and_not_a_cookie_only_source`: KV held `body.example.com` and also `cookie-only.example.com` +- `auction::endpoints::tests::disabled_auction_ingests_nothing_from_the_cookie_but_keeps_shared_id`: KV held `cookie-only.example.com` next to `sharedid.org` +- `navigation_ingests_the_cookie_only_source`, `finalize_generated_ec_without_sync_source_ingests_the_ts_eids_cookie`, the edited `finalize_recovers_orphaned_ec_when_purpose_four_denial_empties_updates` and `auction_route_carries_the_raw_ts_eids_cookie_to_finalization` pass against the old code (regression nets) +- Revert proof: with `browser_eid_input`'s fallback arm changed to `_ => (None, ec_context.client_eids()),`, `navigation_ingests_the_cookie_only_source`, `finalize_generated_ec_without_sync_source_ingests_the_ts_eids_cookie` and `finalize_recovers_orphaned_ec_when_purpose_four_denial_empties_updates` each FAIL (wasm abort); arm restored +- Revert proof: with the Fastly adapter's `let eids_cookie = crate::extract_cookie_value(req, COOKIE_TS_EIDS);` replaced by `let eids_cookie: Option = None;`, `auction_route_carries_the_raw_ts_eids_cookie_to_finalization` FAILS (wasm abort); line restored + +## Task 6: Pure EID collector and cookie value builder + +Command (against a stub `eids.ts` whose functions return `undefined` and an empty cookie): + +```bash +cd crates/trusted-server-js/lib && mise exec node@24.12.0 -- npx vitest run test/integrations/prebid/eids.test.ts +``` + +Result: 22 failed, 9 passed (31 tests). The 9 passes are the four `returns undefined for …` cases, the four `… not an object` cases and `does not modify the entries it is given`, which the stub passes trivially. + +Failures: + +- `collectBidderEids` tests (`reads ortb2.user.ext.eids`, `reads ortb2.user.eids, …`, `reads user.ext.eids before user.eids`, `returns plain-data copies without undefined fields`, `drops an entry whose serialization throws …`, `keeps a set that fits the UTF-8 budget exactly`, `drops the entry that crosses the budget …`, `measures the budget in UTF-8 bytes …`, `keeps order and repeated sources`, `forwards a lone surrogate unchanged …`, `passes the Prebid module shapes from spec 3.5 …`): `expected undefined to deeply equal [...]` +- `fitEidCookie` tests (`removes EID-level ext and drops the duplicates …`, `trims UIDs from the last source …`, `reports a source trimmed and then dropped only as dropped`, both `removes a last entry that is … whole` cases, `removes a null last entry whole …`, `encodes non-Latin-1 text as UTF-8 base64 …`): `should produce a cookie value: expected undefined to be defined` +- `fitEidCookie > drops whole tail sources until the cookie fits`: `TypeError: Cannot read properties of undefined (reading 'length')` +- `fitEidCookie > returns no value when nothing fits`: `droppedSources` is `[]` instead of `['huge.example.com']` +- `fitEidCookie > matches btoa output for ASCII payloads`, `fitEidCookie > produces exactly the ts-eids-cookie fixture for the fixture input`: `expected undefined to be ''` + +## Task 7: Adapter forwards the bidder request's EIDs and owns ts-eids + +Command (canary, helpers and new tests installed; adapter unchanged): + +```bash +cd crates/trusted-server-js/lib && mise exec node@24.12.0 -- npx vitest run test/integrations/prebid/index.test.ts test/core/request.test.ts +``` + +Result: 175 failed, 132 passed. Besides the new tests, about 160 existing tests that build a `trustedServer` request or run `bidsBackHandler` fail in the shared `afterEach`, because the old collector forwards the `canary.example.com` EID from `getUserIdsAsEids()` (`the getUserIdsAsEids canary leaked into an /auction body`) or writes it to `ts-eids`. + +Failures: + +- `forwards an opaque LiveRamp envelope from the bidder request unchanged`: `expected [ { …(2) } ] to deeply equal [ { source: 'liveramp.com', …(1) } ]` (the canary arrives instead) +- `forwards bidderRequest.ortb2 EIDs with EID-level fields, user.ext.eids first`: `should forward both paths as sent: expected [ { …(2) } ] to deeply equal [ …(2) ]` +- `sends exactly the auction-body fixture for the fixture input`: `TSJS should produce the Rust fixture byte for byte` +- `writes exactly the ts-eids fixture for the fixture input`: `expected undefined to be 'W3sic291cmNl…'` +- `writes ts-eids from the body set without EID-level ext`: `the body keeps EID-level ext: expected [ { …(2) } ] to deeply equal [ { …(3) }, { …(2) } ]` +- `clears ts-eids before the request when the bidder request holds no EIDs`: `the body should carry no eids: expected [ { …(2) } ] to be undefined` +- `sends no eids and leaves ts-eids alone without a bidder request object`: `expected [ { …(2) } ] to be undefined` +- `lets the later of two overlapping requests own ts-eids`: `expected undefined to deeply equal [ { …(2) } ]` +- `forwards ortb2 EIDs and writes ts-eids without the User ID module`: `expected undefined to deeply equal [ { source: 'id5-sync.com', …(3) } ]` +- `logs EID counts and the cookie decision at debug level, never values`: `expected "debug" to be called with arguments: [ …(2) ]` +- `keeps the trim log when the cookie had to drop sources`: `the body keeps every entry: expected [ { …(2) } ] to deeply equal [ …(7) ]` +- `clears ts-eids when tcf2Enforcement lists trustedServer in biddersBlocked` and `… in eidsBlocked`, and `leaves ts-eids alone when tcf2Enforcement lists only other bidders`: `the shim should listen for tcf2Enforcement: expected undefined to be defined` +- `leaves ts-eids alone when an auction builds no trustedServer request`: `only buildRequests may write or clear the cookie: expected 'W3sic291cmNl…' to be 'preset-value'` +- `exposes the getUserIdsAsEids canary on the pbjs the shim uses` and `sends no eids in the /auction body`: PASS (positive control and net) +- Net proof: with `src/core/request.ts:45` changed to pass `{ eids: [{ source: 'net.example.com', … }] }` to `buildAdRequest`, `sends no eids in the /auction body` FAILS (`requestAds should leave EIDs to the ts-eids cookie: expected { adUnits: […], …(2) } to not have property "eids"`); line restored + +## Task 8: Stop writing Prebid userSync settings + +Command (new self-init tests added; `installUserIdModules` still present): + +```bash +cd crates/trusted-server-js/lib && mise exec node@24.12.0 -- npx vitest run test/integrations/prebid/index.test.ts -t "writes no userSync|preserves publisher userSync" +``` + +Result: 2 failed, 2 passed. The file-level `afterAll` canary also reports `the canary should have inspected at least one /auction body: expected 0 to be greater than 0`. This comes from the `-t` filter selecting no `/auction` test, not from this change, and a full run satisfies it. + +Failures: + +- `prebid self-init writes no Prebid userSync settings > writes no userSync when the bundle loads after window load`: `TS should leave userSync to the publisher and Prebid: expected [ [ { userSync: { …(6) } } ], …(1) ] to deeply equal []` (the `auctionDelay: 0`, `syncDelay: 3000`, `filterSettings.all` write) +- `prebid self-init writes no Prebid userSync settings > writes no userSync when window load fires after install`: same assertion, same write after the `load` event +- `preserves publisher userSync and autoRefresh=%s when seeding late IDs` (both `false` and `true`): PASS with the extended `syncDelay` and `filterSettings` fields. This is the seeding net; its proof follows in Task 8's green run. + +Net proof (seeding path): with `src/integrations/prebid/index.ts` line 2181 changed from `...(isRecord(effectiveUserSync) ? effectiveUserSync : {}),` to `...{},`, `-t "preserves publisher userSync"` FAILS on both cases: `preserves publisher userSync and autoRefresh=false when seeding late IDs` and `… autoRefresh=true …`, with `expected { userIds: [ { …(3) } ], …(1) } to deeply equal { userIds: [ { …(3) } ], …(5) }`. Line restored; the full file then passes 294 of 294. + +## Task 9: Real Prebid.js arms for D1, D3, D4 and TCF + +Command (the shim built from `main`'s sources: `src/integrations/prebid/index.ts` and `src/core/auction.ts` checked out from the merge base `a9037770a` for one run, then restored with `git checkout --`; `git status --short crates/trusted-server-js/lib/src` was clean afterwards): + +```bash +cd crates/trusted-server-js/lib && mise exec node@24.12.0 -- npx vitest run test/prebid-eid-fidelity.test.mjs +``` + +Result: 5 failed, 1 passed (6 tests). Each failure is the one spec 7.5 predicts. + +Failures: + +- `D1: never forwards an ID restricted to another bidder`: `the /auction body should carry the control and not the restricted ID: expected [ [ { source: 'pubcid.org', …(1) } ] ] to deeply equal [ [ { …(2) } ] ]` (the precondition passed: sharedId produced `pubcid.org` from the preset `_sharedid` cookie) +- `D3: forwards same-source entries separately with their own EID-level fields`: `expected [ undefined ] to deeply equal [ [ { …(5) }, { …(5) } ] ]` (`main` never reads `ortb2`) +- `D4: the first auction after window.load waits for a pending User ID module`: `Prebid's default auctionDelay should apply: expected +0 to be 500` +- `TCF: without an exception no client EIDs reach TS and ts-eids is cleared`: `ts-eids should be cleared: expected [ { source: 'liveramp.com', …(1) } ] to be undefined` (the positive control and the no-`/auction` check passed; `main`'s `bidsBackHandler` rewrote the cookie) +- `TCF: with the full basicAds rule TS gets its EIDs and other bidders stay blocked`: `expected [ Array(1) ] to deeply equal [ [ { …(2) }, { …(2) } ] ]` (the body carried `liveramp.com` only) +- `D4 control: the first auction before window.load waits, on every build`: PASS, as intended + +D4 mechanism probe: a temporary copy of the suite without the `auctionDelay` assertion, run with `-t "D4"` against the same `main` sources, still fails the D4 arm at the body: `the first /auction should carry the delayed unifiedId: expected [ undefined ] to deeply equal [ Array(1) ]`. So on `main` the first auction after `window.load` goes out without `adserver.org`, not merely with a different setting. The copy was deleted and the sources restored. + +## Task 10: Delete the dead EID-to-module resolver + +Command: + +```bash +cd crates/trusted-server-js/lib && mise exec node@24.12.0 -- npx vitest run test/integrations/prebid/user_id_modules.test.ts -t "no EID source metadata" +``` + +Failures: + +- `carries no EID source metadata`: `no entry should keep eidSources: expected [ 'connectIdSystem', …(10) ] to deeply equal []` (11 modules still carry `eidSources`) + +Green: after deleting `eidSources` and the resolver, `carries no EID source metadata` passes. The three targeted suites (`user_id_modules.test.ts`, `prebid-user-id-registry.test.mjs`, `build-prebid-external.test.mjs`) pass 81 of 81. `./scripts/test-cli.sh` exits 0. The full Vitest run under Node 24.12.0 passes 1249 of 1249 across 52 files. + +## Final review fixes + +### Fix 1: nothing fits clears the cookie (regression net) + +Revert proof: temporarily deleted the `clearPrebidEidsCookie();` call inside the `fitted.value === undefined` branch of `writePrebidEidsCookie`, then restored it. + +Command: + +```bash +cd crates/trusted-server-js/lib && mise exec node@24.12.0 -- npx vitest run test/integrations/prebid/index.test.ts -t "no EID fits" +``` + +Failures: + +- `clears ts-eids when no EID fits the cookie budget but still sends the body`: `the cookie should be cleared: expected 'preset-value' to be undefined` + +Green: with the call restored, the test passes. + +### Fix 2: cookie writes must not throw out of buildRequests + +Command (against the unguarded `clearPrebidEidsCookie`): + +```bash +cd crates/trusted-server-js/lib && mise exec node@24.12.0 -- npx vitest run test/integrations/prebid/index.test.ts -t "document.cookie writes throw" +``` + +Failures: + +- `still builds the request with the entry when the cookie write fails`: `Error: cookies blocked` thrown from `clearPrebidEidsCookie` through `buildRequests` +- `still builds the request without eids when the cookie clear fails`: `Error: cookies blocked` thrown from `clearPrebidEidsCookie` through `buildRequests` + +Green: with the `try`/`catch` guard both pass; full Vitest passes 1252 of 1252. + +### Fix 3: non-array cookie JSON is a parse error + +Command (against the unchanged decoder): + +```bash +cargo test-fastly -- parse_prebid_eids_cookie_rejects_non_array_json +cargo test-fastly -- eids_lookup_reports_non_array +``` + +Failures: + +- `parse_prebid_eids_cookie_rejects_non_array_json`: test trapped, the `{"a":1}` cookie parsed as `Ok` +- `eids_lookup_reports_non_array_cookie_as_parse_error`: test trapped, no `parse_error` reported + +Green: with the array check both pass.