diff --git a/.cargo/config.toml b/.cargo/config.toml index ecfed438e2..b88693a6c2 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-shared-51degrees -p trusted-server-middleware-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-shared-51degrees -p trusted-server-middleware-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index b9fb20e06e..11c961e506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,6 +119,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- A 51Degrees client hints middleware, `51degrees.client_hints`, from `crates/middleware/51degrees`, run by `[shared.51degrees]` and placed by a `[[serve]]` entry. On the Fastly adapter, the only one that runs request filters, it asks the 51Degrees service which client hints the reader's browser can send for each page navigation an entry covers, sending the `User-Agent` and `Sec-CH-UA` hints alone, and leaves the hint names in the request's state for the other 51Degrees modules, which keeps a page it names hints for out of the stored templates and sends it `private, no-store`. On each page an entry covers it writes `Accept-CH` and `Critical-CH` on the response unless the browser already sends every one of those hints, `Permissions-Policy` and the `Delegate-CH` tag that delegate those hints to the service's origin, each keeping every line the response already carries. `critical` in `[middleware.51degrees.client_hints]` turns the retry of the first navigation off. The service's answers load through the cache `51degrees-answers`, selected and sized in `[cache.51degrees-answers]`, and are held only where the build links a loading cache module, so a stock build asks the service for each page navigation an entry covers. What the 51Degrees modules share, being the table, the one call to the service, the evidence it sends and the cache its answer is held in, is the new `crates/shared/51degrees`. - `[middleware.]`, a middleware's own settings table, written with the middleware's full name and read through `Settings::middleware_config`. A table holds values and arrays, a table within one is the longer name's table and never a setting, and a table at a name no running module supplies is refused at startup with the names that could be written. - A serve middleware may set headers on a response the publisher path sends for a request path its entry covers, through `Middleware::response_headers`, which is given the response's status and headers and runs whatever the response's media type and whether or not it has a body, so a redirect with no body reaches a middleware that rewrites its `Location`. A header it sets replaces one of the same name. The method is not called for a refusal made before the origin is asked, such as the `405` for a method a signed origin does not take, for a page served from the template cache or the refusal of such a page, or for an answer made from an error. - A publisher's site may be served from a folder of object storage through the publisher proxy, so its pages pass through the page pipeline, which an asset route skips. The folder is the path of `origin_url`, and two optional `[publisher]` settings do the rest, each off unless set. `origin_auth` takes the table an asset route's `auth` takes and signs every origin request with it, through the same signer. `directory_index` names a file added to a request path ending in `/`, for the origin request only. A signed origin is only read, so a method other than `GET` or `HEAD` is answered `405`, and a signed request carries none of the reader's cookies, credentials, referrer, conditional validators or range. A malformed `origin_auth` or `directory_index` is refused when the settings load, with the setting named. diff --git a/Cargo.lock b/Cargo.lock index cc0f8bdf78..7d26047f33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6658,6 +6658,7 @@ dependencies = [ "trusted-server-framework-nextjs", "trusted-server-headers-geo", "trusted-server-identity-lockr", + "trusted-server-middleware-51degrees", "trusted-server-permission-signal-gpc", "trusted-server-permission-signal-gpp", "trusted-server-permission-signal-mtm", @@ -6703,6 +6704,7 @@ dependencies = [ "trusted-server-headers-geo", "trusted-server-identity-lockr", "trusted-server-js", + "trusted-server-middleware-51degrees", "trusted-server-permission-signal-gpc", "trusted-server-permission-signal-gpp", "trusted-server-permission-signal-mtm", @@ -6754,6 +6756,7 @@ dependencies = [ "trusted-server-geo-fastly", "trusted-server-headers-geo", "trusted-server-identity-lockr", + "trusted-server-middleware-51degrees", "trusted-server-permission-signal-gpc", "trusted-server-permission-signal-gpp", "trusted-server-permission-signal-mtm", @@ -6803,6 +6806,7 @@ dependencies = [ "trusted-server-headers-geo", "trusted-server-identity-lockr", "trusted-server-js", + "trusted-server-middleware-51degrees", "trusted-server-permission-signal-gpc", "trusted-server-permission-signal-gpp", "trusted-server-permission-signal-mtm", @@ -7028,6 +7032,7 @@ dependencies = [ "trusted-server-framework-nextjs", "trusted-server-headers-geo", "trusted-server-identity-lockr", + "trusted-server-middleware-51degrees", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", "trusted-server-testing-testlight", @@ -7268,6 +7273,24 @@ dependencies = [ "web-time", ] +[[package]] +name = "trusted-server-middleware-51degrees" +version = "0.1.0" +dependencies = [ + "async-trait", + "edgezero-core", + "error-stack", + "http", + "serde", + "serde_json", + "tokio", + "trusted-server-core", + "trusted-server-loading-cache-51degrees", + "trusted-server-shared-51degrees", + "url", + "validator", +] + [[package]] name = "trusted-server-openrtb" version = "0.1.0" @@ -7342,6 +7365,26 @@ dependencies = [ "trusted-server-permission-signal-gpc", ] +[[package]] +name = "trusted-server-shared-51degrees" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "edgezero-core", + "error-stack", + "http", + "log", + "serde", + "serde_json", + "sha2 0.11.1", + "tokio", + "trusted-server-core", + "trusted-server-loading-cache-51degrees", + "url", + "validator", +] + [[package]] name = "trusted-server-tag-google-tag" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index d24029afd2..9d81ade1fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,8 @@ members = [ "crates/ads-txt/iabtl", "crates/headers/geo", "crates/loading-cache/51degrees", + "crates/shared/51degrees", + "crates/middleware/51degrees", "crates/asset-cache/51degrees", "crates/tag/google-tag", "crates/tag/google-tag-manager", @@ -172,6 +174,8 @@ trusted-server-cmp-inmobi = { path = "crates/cmp/inmobi" } trusted-server-ads-txt-iabtl = { path = "crates/ads-txt/iabtl" } trusted-server-headers-geo = { path = "crates/headers/geo" } trusted-server-loading-cache-51degrees = { path = "crates/loading-cache/51degrees" } +trusted-server-shared-51degrees = { path = "crates/shared/51degrees" } +trusted-server-middleware-51degrees = { path = "crates/middleware/51degrees" } trusted-server-asset-cache-51degrees = { path = "crates/asset-cache/51degrees" } trusted-server-tag-google-tag = { path = "crates/tag/google-tag" } trusted-server-tag-google-tag-manager = { path = "crates/tag/google-tag-manager" } diff --git a/crates/middleware/51degrees/Cargo.toml b/crates/middleware/51degrees/Cargo.toml new file mode 100644 index 0000000000..b4e73d403c --- /dev/null +++ b/crates/middleware/51degrees/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "trusted-server-middleware-51degrees" +description = "The 51Degrees client hints middleware: asks the service which hints a reader's browser can send, and writes Accept-CH, Critical-CH, Permissions-Policy and the Delegate-CH tag on each page." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +trusted-server-core = { workspace = true } +trusted-server-shared-51degrees = { workspace = true } +async-trait = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +url = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt", "macros"] } +trusted-server-core = { workspace = true, features = ["test-utils"] } +trusted-server-loading-cache-51degrees = { workspace = true } +trusted-server-shared-51degrees = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. +[package.metadata.maintainers] +owner = "51Degrees" +status = "vendor owned" diff --git a/crates/middleware/51degrees/README.md b/crates/middleware/51degrees/README.md new file mode 100644 index 0000000000..b1c396a2f0 --- /dev/null +++ b/crates/middleware/51degrees/README.md @@ -0,0 +1,20 @@ +# 51Degrees client hints middleware + +`trusted-server-middleware-51degrees` supplies the `51degrees.client_hints` +serve middleware. On the Fastly adapter, the only adapter that runs request +filters, it asks the 51Degrees service which client hints the reader's +browser can send for each page navigation a `[[serve]]` entry naming it +covers, sending the `User-Agent` and `Sec-CH-UA` hints alone, and leaves the +hint names in the request's state for the other 51Degrees modules. On each +page a `[[serve]]` entry names it for, it writes `Accept-CH` and `Critical-CH` +on the response unless the browser already sends every one of those hints, +and `Permissions-Policy` and the `Delegate-CH` tag that delegate those hints +to the service. + +`[shared.51degrees]` runs it, and `critical` in +`[middleware.51degrees.client_hints]` turns the retry of the first navigation +off. What it shares with the other 51Degrees modules is in +`crates/shared/51degrees`. + +The [51Degrees guide](../../../docs/guide/integrations/51degrees.md) says +how to place and configure it. diff --git a/crates/middleware/51degrees/src/fixtures/page-change.settings.toml b/crates/middleware/51degrees/src/fixtures/page-change.settings.toml new file mode 100644 index 0000000000..0285dada52 --- /dev/null +++ b/crates/middleware/51degrees/src/fixtures/page-change.settings.toml @@ -0,0 +1,7 @@ +[shared.51degrees] +endpoint = "https://cloud.example.com/api/v4/json" +license_key = "a-license-key-for-the-recorded-page" + +[[serve]] +media_type = "text/html" +middleware = ["51degrees.client_hints"] diff --git a/crates/middleware/51degrees/src/lib.rs b/crates/middleware/51degrees/src/lib.rs new file mode 100644 index 0000000000..d2a2e3ef82 --- /dev/null +++ b/crates/middleware/51degrees/src/lib.rs @@ -0,0 +1,1662 @@ +//! The 51Degrees client hints middleware, `51degrees.client_hints`. +//! +//! A browser sends its high entropy client hints only when the site asks for +//! them, with `Accept-CH` on the response and `Critical-CH` when the first +//! navigation should be retried carrying them, and they reach a third party +//! only when the top level page delegates them, by a `Delegate-CH` meta tag +//! or a `Permissions-Policy` header. A `Delegate-CH` tag added by JavaScript +//! is ignored, so the tag has to be in the HTML as served. +//! +//! No hint name is written down here. [`ClientHints`] asks the service, +//! through the shared crate and its cache, which hints it wants from the +//! request's browser as it filters a request for a page a `[[serve]]` entry +//! names it for, leaves the names in the request's state, and writes them on +//! that page, the first request included: the `Delegate-CH` tag in the head, +//! and `Accept-CH`, `Critical-CH` and `Permissions-Policy` on the response, +//! each keeping every line the response already carries under its name. The +//! tag's content follows 51Degrees' published example, lower case hint +//! names, a space and the origin, separated by semicolons. +//! +//! `Critical-CH` is written when `critical` in +//! `[middleware.51degrees.client_hints]` is on, which it is unless the table +//! turns it off. +//! +//! A browser keeps an origin's `Accept-CH` set until a response names +//! another, so `Accept-CH` and `Critical-CH` are left off the response to a +//! request that already carries every hint the service names, unless the +//! response names a set of its own, which would replace the one held. The tag +//! and `Permissions-Policy` apply to their own page only, and are written on +//! every page. + +use std::sync::Arc; + +use async_trait::async_trait; +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::evidence::{BorrowedRequestInfo, RequestInfo}; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationDocumentState, IntegrationRegistration, + IntegrationRequestFilter, IntegrationRequestState, ModuleSecretSetting, RequestFilterDecision, + RequestFilterEffects, RequestFilterInput, SuppliedMiddleware, +}; +use trusted_server_core::middleware::{ + HTML_MEDIA_TYPE, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, + ResponseHeader, ResponseView, +}; +use trusted_server_core::platform::RuntimeServices; +use trusted_server_core::redacted::Redacted; +use trusted_server_core::settings::Settings; +use trusted_server_shared_51degrees::client::CloudPipeline; +use trusted_server_shared_51degrees::config::{self, SharedConfig}; +use trusted_server_shared_51degrees::engine_cache::{AnswerCache, CACHE}; +use trusted_server_shared_51degrees::hints_engine::{ + ACCEPT_CH_VALUES, ClientHintsCall, ServiceHints, hint_names, +}; +use trusted_server_shared_51degrees::license::license_key; +use validator::Validate; + +/// The id the 51Degrees registration carries, under which the filter leaves +/// the hint names in the request's state. +pub use trusted_server_shared_51degrees::BUILDER_ID; + +/// The name a `[[serve]]` entry selects the client hints middleware by, and +/// the name of its own table under `[middleware]`, from this crate's folder +/// with the entry after it. +pub const MODULE: &str = "51degrees.client_hints"; + +/// The properties the client hints middleware reads from the one answer the +/// deployment's pipeline holds for a request. +pub use trusted_server_shared_51degrees::hints_engine::ACCEPT_CH_VALUES as PROPERTIES; + +/// The license key, named in `[shared.51degrees]` by its entry in the secret +/// store, which is in use whenever the table is present. +const SECRET_SETTINGS: &[ModuleSecretSetting] = &[ModuleSecretSetting { + path: &["license_key"], + in_use: always_in_use, +}]; + +fn always_in_use(_table: &serde_json::Map) -> bool { + true +} + +/// The cache the service's answers are held in, which a deployment selects +/// and sizes in `[cache.51degrees-answers]`. +const CACHE_FEATURES: &[&str] = &[CACHE]; + +/// The page change the module supplies, as a `[[serve]]` entry names it. +const SUPPLIED_MIDDLEWARE: &[SuppliedMiddleware] = + &[SuppliedMiddleware::on_html(MODULE, MiddlewarePhase::Serve)]; + +/// The builder a deployment installs to offer this middleware, which +/// `[shared.51degrees]` runs. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(BUILDER_ID, env!("CARGO_PKG_NAME"), register, validate) + .with_shared_settings(config::vendor()) + .with_secret_settings(SECRET_SETTINGS) + .with_cache_features(CACHE_FEATURES) + .with_middleware(SUPPLIED_MIDDLEWARE) +} + +/// Registers the client hints middleware and its request filter when +/// `[shared.51degrees]` is present. +/// +/// # Errors +/// +/// When the table cannot be read, has no license key, or names an endpoint +/// the cloud calls cannot be built on, or when the middleware's own table +/// cannot be read. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(shared) = config::read(settings)? else { + return Ok(None); + }; + // Fatal when absent, whatever the modules are used for. See `license_key`. + let license_key = license_key(settings)?; + let pipeline = pipeline(&shared, &license_key, settings)?; + let middleware = Arc::new(ClientHints::new( + &shared.endpoint, + ClientHintsCall::on(pipeline), + critical_from_settings(settings)?, + )); + Ok(Some( + IntegrationRegistration::builder(BUILDER_ID) + // The headers and the tag are the whole client side. + .without_js() + .with_request_filter(Arc::clone(&middleware) as Arc) + .with_middleware(middleware) + .build(), + )) +} + +/// Validates the configuration for deployment and reports whether +/// `[shared.51degrees]` runs this middleware. +/// +/// # Errors +/// +/// The same refusals as [`register`], so a deployment with no license key is +/// stopped by the configuration check rather than only when the state is +/// built. +pub(crate) fn validate(settings: &Settings) -> Result> { + let Some(shared) = config::read(settings)? else { + return Ok(false); + }; + let license_key = license_key(settings)?; + pipeline(&shared, &license_key, settings)?; + critical_from_settings(settings)?; + Ok(true) +} + +/// The cloud pipeline the 51Degrees modules share, asking for the properties +/// this middleware reads, with its answers held in the cache +/// `[cache.51degrees-answers]` selects. +fn pipeline( + shared: &SharedConfig, + license_key: &Redacted, + settings: &Settings, +) -> Result, Report> { + CloudPipeline::new( + &shared.endpoint, + license_key.expose(), + shared.timeout_ms, + AnswerCache::for_settings(settings), + ) + .map(|pipeline| Arc::new(pipeline.wanting(ACCEPT_CH_VALUES))) + .ok_or_else(|| { + Report::new(TrustedServerError::Configuration { + message: format!( + "[shared.{}] endpoint `{}` is not an address the cloud calls can be built on", + config::vendor(), + shared.endpoint + ), + }) + }) +} + +/// `[middleware.51degrees.client_hints]`, the middleware's own options. +#[derive(Debug, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +struct ClientHintsSettings { + /// Whether `Critical-CH` is written with `Accept-CH`. + #[serde(default = "default_true")] + critical: bool, +} + +const fn default_true() -> bool { + true +} + +/// Whether `Critical-CH` is written, from `[middleware.51degrees.client_hints]`, +/// which is on unless the table says `critical = false`. +/// +/// # Errors +/// +/// When the table carries a key it does not know or a value that is not +/// `true` or `false`. +pub fn critical_from_settings(settings: &Settings) -> Result> { + Ok(settings + .middleware_config::(MODULE)? + .is_none_or(|options| options.critical)) +} + +/// The client hints delegation, written on each page a `[[serve]]` entry +/// names it for. +/// +/// Serve only, because the list is the service's answer for one reader's +/// browser, and what the fetch phase leaves is stored for every reader. +#[derive(Debug)] +pub struct ClientHints { + /// The origin the hints are delegated to, for example + /// `https://cloud.example.com`. Derived from the configured endpoint, so + /// a deployment using its own private cloud delegates to that instead. + endpoint_origin: Option, + /// Asks which hints the service wants from the reader's browser, sending + /// its `User-Agent` and `Sec-CH-UA` hints alone, through the cache, which + /// holds one answer for each set of those values and so answers every + /// reader on the same browser after the first. + hints: ClientHintsCall, + /// Whether `Critical-CH` is written with `Accept-CH`, so the browser + /// retries the first navigation carrying its hints. On by default, + /// because without it the first page view of a session is priced on a + /// User-Agent alone, at the cost of one extra origin fetch at the start + /// of a session. The retry cannot loop, because a browser retries only + /// when the named hints were absent and the retried request carries them. + critical: bool, +} + +impl ClientHints { + /// Creates the middleware for a configured endpoint and the shared + /// crate's client hints question. An endpoint that is not a URL, or has + /// no host, yields a middleware that writes nothing. Startup validation + /// refuses such an endpoint, so this is only a fallback. + #[must_use] + pub fn new(endpoint: &str, hints: ClientHintsCall, critical: bool) -> Self { + Self { + endpoint_origin: endpoint_origin(endpoint), + hints, + critical, + } + } + + /// The names the service asks for, for the request's browser, or nothing + /// when the service names nothing or did not answer. The call logs why. + async fn names( + &self, + request_info: &dyn RequestInfo, + services: &RuntimeServices, + ) -> Vec { + match self.hints.accept_ch(request_info, services).await { + Some(accept_ch) => hint_names(&accept_ch), + None => Vec::new(), + } + } +} + +/// Left beside the names when the request already carried every one of them, +/// so the browser holds them for the origin. +#[derive(Debug)] +struct HintsHeld; + +/// The names the filter left for this request, which core copies into the +/// document state from the request's state on serve, or none for a document +/// with no request behind it. +fn names_in(document_state: &IntegrationDocumentState) -> Vec { + document_state + .get::(BUILDER_ID) + .map_or_else(Vec::new, |hints| hints.names().to_vec()) +} + +impl Middleware for ClientHints { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == HTML_MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + /// Writes the delegation tag in the head, for the names the filter left + /// for this request. A document with no names, or a fetch-phase + /// document, gets nothing. + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let head_inserts = delegation_for( + self.endpoint_origin.as_deref(), + context.request_host, + &names_in(context.document_state), + ) + .into_iter() + .collect(); + MiddlewareAction { + head_inserts, + ..MiddlewareAction::pass() + } + } + + /// Writes `Accept-CH`, `Critical-CH` and `Permissions-Policy` for the + /// names the filter left for this request, each keeping what the + /// response already carries, with the first two left off where the + /// browser holds the names already. The headers ask the browser to send + /// the hints with its later requests as well as with the ones the tag + /// covers. + fn response_headers( + &self, + context: &MiddlewareContext<'_>, + response: &ResponseView<'_>, + ) -> Vec { + client_hint_headers( + &names_in(context.document_state), + self.critical, + context + .document_state + .get::(BUILDER_ID) + .is_some(), + delegated_origin(self.endpoint_origin.as_deref(), context.request_host), + response.headers, + ) + } +} + +/// Whether `request` is a navigation to a page, which is what the delegation +/// is for. An asset or an API call gets no call to the service. +fn asks_for_a_document(request: &http::Request) -> bool { + if request.method() != http::Method::GET && request.method() != http::Method::HEAD { + return false; + } + let header = |name: &str| { + request + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + }; + header("sec-fetch-dest").eq_ignore_ascii_case("document") + || header("accept").to_ascii_lowercase().contains("text/html") +} + +/// Whether a `[[serve]]` entry whose path covers `path` names this +/// middleware, by the rule the headers step uses, so the service is asked +/// only for a page whose response the middleware writes on. +fn placed_for(settings: &Settings, path: &str) -> bool { + settings + .phase_entries(MiddlewarePhase::Serve) + .for_path(path) + .contains(&MODULE) +} + +#[async_trait(?Send)] +impl IntegrationRequestFilter for ClientHints { + fn integration_id(&self) -> &'static str { + BUILDER_ID + } + + /// Asks the service, through the cache, which hints it wants from this + /// browser, and leaves their names in the request's state for the + /// document. A request that is not a page navigation, one whose path no + /// `[[serve]]` entry naming this middleware covers, and a service that + /// names nothing or does not answer, leave nothing, and the page gets no + /// tag and no header. + async fn filter_request( + &self, + input: RequestFilterInput<'_>, + ) -> Result> { + if asks_for_a_document(input.request) + && placed_for(input.settings, input.request.uri().path()) + { + let names = { + // No address, because the hints depend on the browser alone. + let evidence = BorrowedRequestInfo::new("", Some(input.request.headers())); + self.names(&evidence, input.services).await + }; + if !names.is_empty() { + if carries_every(input.request.headers(), &names) { + IntegrationRequestState::insert(input.request, BUILDER_ID, HintsHeld); + } + IntegrationRequestState::insert( + input.request, + BUILDER_ID, + ServiceHints::new(names), + ); + } + } + Ok(RequestFilterDecision::Continue( + RequestFilterEffects::default(), + )) + } +} + +/// The names among `hints` that are tokens, being lower case letters, digits +/// and hyphens. A name that is not one never reaches a header or the tag, +/// because the tag's content sits inside a double quoted attribute. +fn tokens(hints: &[String]) -> Vec<&str> { + hints + .iter() + .map(String::as_str) + .filter(|hint| { + !hint.is_empty() + && hint + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + }) + .collect() +} + +/// Whether `headers` carry every one of `names`. A browser sends a high +/// entropy hint only once a response from the origin has named it in +/// `Accept-CH`, and the low entropy ones always, so only the whole list shows +/// the set is held. A header counts whatever its value, because +/// `Sec-CH-UA-Model` is an empty string on a desktop. +fn carries_every(headers: &http::HeaderMap, names: &[String]) -> bool { + !names.is_empty() && names.iter().all(|name| headers.contains_key(name.as_str())) +} + +/// `Accept-CH` naming `hints`, with `Critical-CH` naming the same when +/// `critical` is on, and `Permissions-Policy` delegating each to `origin` +/// when there is one, or nothing for no hints. +/// +/// Each keeps every line `existing` carries under that name, joined into one +/// list, with the hints added after it, because core replaces all of those +/// lines with the one header written here. So a policy the publisher's origin +/// sets is kept and the hints it already asks for are still asked for. +/// `Accept-CH` and `Critical-CH` name the same hints, because a browser +/// retries only for the hints it was told are critical. Both are left off +/// when `held`, unless `existing` names an `Accept-CH` set of its own, which +/// would replace the one the browser holds. +fn client_hint_headers( + hints: &[String], + critical: bool, + held: bool, + origin: Option<&str>, + existing: &http::HeaderMap, +) -> Vec { + let hints = tokens(hints); + if hints.is_empty() { + return Vec::new(); + } + let said = |name: &str| { + let lines: Vec<&str> = existing + .get_all(name) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect(); + (!lines.is_empty()).then(|| lines.join(", ")) + }; + let mut headers = Vec::new(); + let mut write = |name: &'static str, value: String| { + if let Ok(value) = http::HeaderValue::from_str(&value) { + headers.push((http::HeaderName::from_static(name), value)); + } + }; + let accept_ch = said("accept-ch"); + if !held || accept_ch.is_some() { + write("accept-ch", with_hints(accept_ch.as_deref(), &hints)); + if critical { + write( + "critical-ch", + with_hints(said("critical-ch").as_deref(), &hints), + ); + } + } + if let Some(origin) = origin { + write( + "permissions-policy", + with_delegation(said("permissions-policy").as_deref(), origin, &hints), + ); + } + headers +} + +/// The comma separated list `existing` holds with each of `hints` it does +/// not already name added after it. +fn with_hints(existing: Option<&str>, hints: &[&str]) -> String { + let mut names: Vec<&str> = existing + .map(|value| { + value + .split(',') + .map(str::trim) + .filter(|name| !name.is_empty()) + }) + .into_iter() + .flatten() + .collect(); + for hint in hints { + if !names.iter().any(|name| name.eq_ignore_ascii_case(hint)) { + names.push(*hint); + } + } + names.join(", ") +} + +/// The policy `existing` holds with a delegation of each of `hints` to +/// `origin` added after it, as `ch-ua-model=(self "https://cloud.example.com")`, +/// which keeps the hint for the page's own origin as well. In a policy the +/// later member of a name wins, so the delegation stands where the +/// publisher's own policy named the hint. +fn with_delegation(existing: Option<&str>, origin: &str, hints: &[&str]) -> String { + let mut members: Vec = existing + .map(|value| { + value + .split(',') + .map(str::trim) + .filter(|member| !member.is_empty()) + }) + .into_iter() + .flatten() + .map(str::to_owned) + .collect(); + for hint in hints { + members.push(format!(r#"{}=(self "{origin}")"#, policy_feature(hint))); + } + members.join(", ") +} + +/// The `Permissions-Policy` feature that governs the hint `hint`, which is +/// the hint's name without its `sec-` prefix, under `ch-`: `sec-ch-ua-model` +/// is `ch-ua-model`, and a legacy hint such as `dpr` is `ch-dpr`. +fn policy_feature(hint: &str) -> String { + let name = hint.strip_prefix("sec-").unwrap_or(hint); + if name.starts_with("ch-") { + name.to_owned() + } else { + format!("ch-{name}") + } +} + +/// The origin part of `endpoint`, with the port where there is one. +fn endpoint_origin(endpoint: &str) -> Option { + url::Url::parse(endpoint).ok().and_then(|url| { + let host = url.host_str()?; + let scheme = url.scheme(); + Some(url.port().map_or_else( + || format!("{scheme}://{host}"), + |port| format!("{scheme}://{host}:{port}"), + )) + }) +} + +/// The origin the hints are delegated to from a page served on +/// `request_host`, or nothing when there is no endpoint or when the endpoint +/// is the page's own origin, which leaves no third party to delegate to. +fn delegated_origin<'a>(origin: Option<&'a str>, request_host: &str) -> Option<&'a str> { + let origin = origin?; + let same_origin = origin + .strip_prefix("https://") + .or_else(|| origin.strip_prefix("http://")) + .is_some_and(|host| host.eq_ignore_ascii_case(request_host)); + (!same_origin).then_some(origin) +} + +/// The tag delegating `hints` from a page served on `request_host`, or nothing +/// when there is no origin to delegate to or no hint to delegate. +/// +/// No hints means the service named none for this browser or has not +/// answered, and the tag is not guessed at. +fn delegation_for(origin: Option<&str>, request_host: &str, hints: &[String]) -> Option { + let origin = delegated_origin(origin, request_host)?; + let pairs: Vec = tokens(hints) + .iter() + .map(|hint| format!("{hint} {origin}")) + .collect(); + if pairs.is_empty() { + return None; + } + Some(format!( + r#""#, + pairs.join("; ") + )) +} + +#[cfg(test)] +mod tests { + use edgezero_core::body::Body; + use trusted_server_core::html_processor::test_support::page_settings; + use trusted_server_core::integrations::{ + IntegrationDocumentState, IntegrationRegistry, RequestFilterInput, + }; + use trusted_server_core::middleware::{MiddlewareContext, Prepared, test_support}; + use trusted_server_shared_51degrees::test_services::{ + MemoryKvStore, ScriptedHttpClient, services_with, + }; + + use super::*; + + const CLOUD: &str = "https://cloud.example.com/api/v4/json"; + const CLOUD_ORIGIN: &str = "https://cloud.example.com"; + + /// The names the live service's `Accept-CH` gives a publisher's license + /// key for Chrome, lower case. + const SERVICE_NAMES: &[&str] = &[ + "sec-ch-ua", + "sec-ch-ua-full-version-list", + "sec-ch-ua-mobile", + "sec-ch-ua-platform", + "sec-ch-ua-model", + "sec-ch-ua-platform-version", + ]; + + /// What the data endpoint answers for a license key naming the three + /// `Accept-CH` properties. + const DATA: &str = r#"{"device":{ + "setheaderbrowseraccept-ch":"Sec-CH-UA,Sec-CH-UA-Platform", + "setheaderhardwareaccept-ch":"Sec-CH-UA-Model", + "setheaderplatformaccept-ch":"Sec-CH-UA-Platform-Version"}}"#; + + /// A deployment's `[shared.51degrees]`, with `rest` after it. + fn settings_with(rest: &str) -> Settings { + page_settings(&format!( + "[shared.51degrees]\nendpoint = \"{CLOUD}\"\nlicense_key = \"a-secret-license-key\"\n\n{rest}" + )) + } + + fn hints_call() -> ClientHintsCall { + ClientHintsCall::new( + CLOUD, + "a-secret-license-key", + 1000, + trusted_server_shared_51degrees::test_services::answer_cache(), + ) + .expect("the endpoint is a URL") + } + + fn middleware(endpoint: &str) -> ClientHints { + ClientHints::new(endpoint, hints_call(), true) + } + + /// The tags the middleware writes for `names`, as the service left them, + /// on a page served on `www.example.com`. + fn delegation(endpoint: &str, names: &[&str]) -> Vec { + let names: Vec = names.iter().map(|name| (*name).to_owned()).collect(); + delegation_for( + endpoint_origin(endpoint).as_deref(), + "www.example.com", + &names, + ) + .into_iter() + .collect() + } + + /// A serve-phase context on `www.example.com` whose document state holds + /// `names` as core seeds them from the request's state, or nothing for an + /// empty list. + fn serve_context<'a>( + state: &'a IntegrationDocumentState, + prepared: &'a Prepared, + names: &[&str], + ) -> MiddlewareContext<'a> { + if !names.is_empty() { + state.get_or_insert_with(BUILDER_ID, || { + ServiceHints::new(names.iter().map(|name| (*name).to_owned()).collect()) + }); + } + MiddlewareContext { + phase: MiddlewarePhase::Serve, + request_host: "www.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + platform: test_support::platform(), + document_state: state, + max_buffered_script_bytes: 1 << 20, + prepared, + } + } + + /// The headers `middleware` sets on a `200` response carrying `existing`, + /// for a document whose state holds `names`, as name and value. + fn headers_for( + middleware: &ClientHints, + names: &[&str], + existing: &[(&'static str, &'static str)], + ) -> Vec<(String, String)> { + headers_with(middleware, names, existing, false) + } + + /// As [`headers_for`], for a document whose request carried every name + /// when `held`. + fn headers_with( + middleware: &ClientHints, + names: &[&str], + existing: &[(&'static str, &'static str)], + held: bool, + ) -> Vec<(String, String)> { + let state = IntegrationDocumentState::default(); + if held { + state.get_or_insert_with(BUILDER_ID, || HintsHeld); + } + let prepared = Prepared::default(); + let context = serve_context(&state, &prepared, names); + let mut headers = http::HeaderMap::new(); + for (name, value) in existing { + headers.append( + http::HeaderName::from_static(name), + http::HeaderValue::from_static(value), + ); + } + let view = ResponseView { + status: http::StatusCode::OK, + headers: &headers, + }; + middleware + .response_headers(&context, &view) + .into_iter() + .map(|(name, value)| { + ( + name.as_str().to_owned(), + value.to_str().unwrap_or_default().to_owned(), + ) + }) + .collect() + } + + fn pairs(headers: &[(&str, &str)]) -> Vec<(String, String)> { + headers + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect() + } + + #[test] + fn module_constant_is_the_crate_folder_with_the_entry() { + assert_eq!( + MODULE, + trusted_server_core::module_name::short_form( + trusted_server_core::settings::MIDDLEWARE_TYPE, + trusted_server_core::module_name!("client_hints") + ), + "should be named by the folder this crate lives in, with the entry after it" + ); + } + + // The headers on a document. + + #[test] + fn the_names_the_filter_left_are_written_as_the_three_headers() { + let headers = headers_for(&middleware(CLOUD), &["sec-ch-ua", "sec-ch-ua-model"], &[]); + + assert_eq!( + headers, + pairs(&[ + ("accept-ch", "sec-ch-ua, sec-ch-ua-model"), + ("critical-ch", "sec-ch-ua, sec-ch-ua-model"), + ( + "permissions-policy", + r#"ch-ua=(self "https://cloud.example.com"), ch-ua-model=(self "https://cloud.example.com")"# + ), + ]), + "should ask the browser for the names the service wants, on this response too, \ + and delegate each to the service" + ); + } + + #[test] + fn critical_off_writes_accept_ch_without_a_retry() { + let headers = headers_for( + &ClientHints::new(CLOUD, hints_call(), false), + &["sec-ch-ua"], + &[], + ); + + assert_eq!( + headers, + pairs(&[ + ("accept-ch", "sec-ch-ua"), + ( + "permissions-policy", + r#"ch-ua=(self "https://cloud.example.com")"# + ), + ]), + "should ask without demanding a retry when critical is off" + ); + } + + #[test] + fn a_document_with_no_names_gets_no_header_and_no_tag() { + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = serve_context(&state, &prepared, &[]); + + assert!( + middleware(CLOUD).create(&context).is_pass(), + "should write nothing for a document the filter left no names for" + ); + assert!(headers_for(&middleware(CLOUD), &[], &[]).is_empty()); + } + + /// What the response already says under each name is kept, with the + /// hints added after it, so the origin's own policy and the hints it + /// asks for on its own survive. + #[test] + fn what_the_response_already_says_is_kept() { + let headers = headers_for( + &middleware(CLOUD), + &["sec-ch-ua-model"], + &[ + ("accept-ch", "Sec-CH-Prefers-Color-Scheme, sec-ch-ua-model"), + ("permissions-policy", "camera=(), ch-ua-model=()"), + ], + ); + + assert_eq!( + headers, + pairs(&[ + ("accept-ch", "Sec-CH-Prefers-Color-Scheme, sec-ch-ua-model"), + ("critical-ch", "sec-ch-ua-model"), + ( + "permissions-policy", + r#"camera=(), ch-ua-model=(), ch-ua-model=(self "https://cloud.example.com")"# + ), + ]), + "a hint the origin already asks for is named once, and the delegation comes \ + after the origin's own member so it is the one that stands" + ); + } + + /// A response may send each header on several lines, and core replaces + /// them all with the one written, so every line is kept, in order. + #[test] + fn every_line_the_response_sends_is_kept() { + let headers = headers_for( + &middleware(CLOUD), + &["sec-ch-ua-model"], + &[ + ("accept-ch", "Sec-CH-Prefers-Color-Scheme"), + ("accept-ch", "Sec-CH-Viewport-Width"), + ("critical-ch", "Sec-CH-Prefers-Color-Scheme"), + ("critical-ch", "Sec-CH-Viewport-Width"), + ("permissions-policy", "camera=()"), + ("permissions-policy", "geolocation=(self)"), + ], + ); + + assert_eq!( + headers, + pairs(&[ + ( + "accept-ch", + "Sec-CH-Prefers-Color-Scheme, Sec-CH-Viewport-Width, sec-ch-ua-model" + ), + ( + "critical-ch", + "Sec-CH-Prefers-Color-Scheme, Sec-CH-Viewport-Width, sec-ch-ua-model" + ), + ( + "permissions-policy", + r#"camera=(), geolocation=(self), ch-ua-model=(self "https://cloud.example.com")"# + ), + ]) + ); + } + + /// A policy feature is the hint's name without `sec-`, under `ch-`. + #[test] + fn a_hint_is_governed_by_its_policy_feature() { + for (hint, feature) in [ + ("sec-ch-ua-model", "ch-ua-model"), + ("sec-ch-ua-full-version-list", "ch-ua-full-version-list"), + ("sec-ch-prefers-color-scheme", "ch-prefers-color-scheme"), + ("dpr", "ch-dpr"), + ("viewport-width", "ch-viewport-width"), + ] { + assert_eq!(policy_feature(hint), feature, "{hint}"); + } + } + + /// An endpoint on the page's own origin leaves no third party to + /// delegate to, so the hints are asked for and nothing is delegated. + #[test] + fn an_endpoint_on_the_pages_own_origin_delegates_nothing() { + let headers = headers_for( + &middleware("https://www.example.com/api/v4/json"), + &["sec-ch-ua-model"], + &[], + ); + + assert_eq!( + headers, + pairs(&[ + ("accept-ch", "sec-ch-ua-model"), + ("critical-ch", "sec-ch-ua-model"), + ]) + ); + } + + // The tag. + + #[test] + fn the_tag_names_every_hint_the_service_asks_for() { + assert_eq!( + delegation(CLOUD, SERVICE_NAMES), + vec![format!( + r#""# + )], + "the service's list, in its order, with nothing added and nothing dropped" + ); + } + + #[test] + fn the_tag_matches_the_shape_the_vendor_publishes() { + let inserts = delegation(CLOUD, &["sec-ch-ua-model", "sec-ch-ua-platform-version"]); + + assert_eq!( + inserts, + vec![format!( + r#""# + )], + "the shape is taken from 51Degrees' own published example, and a browser \ + that does not recognize the content ignores the whole tag silently" + ); + } + + #[test] + fn a_hint_nothing_here_has_heard_of_is_still_delegated() { + // The list is the service's, so a hint it starts asking for is + // delegated the day it does, with no change here. + assert_eq!( + delegation(CLOUD, &["sec-ch-ua-something-new"]), + vec![format!( + r#""# + )], + "a list of what to delegate would have dropped this silently" + ); + } + + #[test] + fn no_answer_from_the_service_writes_no_tag() { + // A service that named nothing for this browser, or did not answer, + // leaves no list, and the tag is not guessed at. + assert!(delegation(CLOUD, &[]).is_empty()); + } + + #[test] + fn a_document_with_no_request_behind_it_writes_nothing() { + // The fetch phase prepares nothing for one reader, and nothing the + // middleware writes may depend on one reader there. + let state = IntegrationDocumentState::default(); + let context = test_support::context(MiddlewarePhase::Fetch, &state); + + assert!(middleware(CLOUD).create(&context).is_pass()); + } + + #[test] + fn a_name_that_is_not_a_token_never_reaches_the_tag_or_a_header() { + // The content sits inside a double quoted attribute, so a name + // carrying a quote would end the attribute early. + let names = ["sec-ch-ua-model\" onload=x", "sec-ch-ua-arch"]; + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = serve_context(&state, &prepared, &names); + + assert_eq!( + middleware(CLOUD).create(&context).head_inserts, + vec![format!( + r#""# + )] + ); + assert_eq!( + headers_for(&middleware(CLOUD), &names, &[])[0], + ("accept-ch".to_owned(), "sec-ch-ua-arch".to_owned()) + ); + } + + /// The tag delegates to the origin of the configured endpoint, so a + /// publisher running a private cloud delegates the hints to that rather + /// than to the public service. An endpoint on the page's own origin leaves + /// no third party to delegate to, and one that is not a URL writes nothing + /// either. + #[test] + fn the_tag_delegates_to_the_configured_endpoints_origin() { + for (case, endpoint, origin) in [ + ("the public cloud", CLOUD, Some(CLOUD_ORIGIN)), + ( + "a private cloud", + "https://cloud.publisher.example/api/v4/json", + Some("https://cloud.publisher.example"), + ), + ( + // An origin missing its port delegates to a different one. + "a private cloud on a port of its own", + "http://127.0.0.1:8080/api/v4/json", + Some("http://127.0.0.1:8080"), + ), + ( + "the page's own origin", + "https://www.example.com/api/v4/json", + None, + ), + ("an endpoint that is not a URL", "not-a-url", None), + ("an endpoint with spaces in it", "not a url", None), + ] { + let expected: Vec = origin + .map(|origin| { + format!(r#""#) + }) + .into_iter() + .collect(); + assert_eq!( + delegation(endpoint, &["sec-ch-ua-model"]), + expected, + "{case}" + ); + } + } + + #[test] + fn the_delegation_runs_on_serve_alone() { + // What the fetch phase leaves is stored for every reader, and the + // list is the service's answer for one reader's browser. + assert_eq!(middleware(CLOUD).phases(), &[MiddlewarePhase::Serve]); + } + + // The filter. + + /// A first page view in Chrome, before anything has asked for a high + /// entropy hint. + fn first_view() -> http::Request { + let mut request = first_view_from("Mozilla/5.0"); + for hint in ["sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform"] { + request.headers_mut().insert( + http::HeaderName::from_static(hint), + http::HeaderValue::from_static("\"something\""), + ); + } + request + } + + /// A first page view from `user_agent`, carrying no client hint at all, + /// which is the request the delegation exists for. + fn first_view_from(user_agent: &'static str) -> http::Request { + edgezero_core::http::request_builder() + .method("GET") + .uri("https://www.example.com/") + .header("host", "www.example.com") + .header("accept", "text/html,application/xhtml+xml") + .header("sec-fetch-dest", "document") + .header("user-agent", user_agent) + .body(Body::empty()) + .expect("should build the request") + } + + /// A request for a script, which the delegation is not for. + fn an_asset() -> http::Request { + edgezero_core::http::request_builder() + .method("GET") + .uri("https://www.example.com/app.js") + .header("host", "www.example.com") + .header("accept", "*/*") + .header("sec-fetch-dest", "script") + .header("user-agent", "Mozilla/5.0") + .body(Body::empty()) + .expect("should build the request") + } + + /// A `[[serve]]` entry naming the middleware on every HTML page. + const PLACED: &str = + "[[serve]]\nmedia_type = \"text/html\"\nmiddleware = [\"51degrees.client_hints\"]\n"; + + /// What the filter leaves in `request`'s state, asked as core asks it, + /// where an entry names the middleware on every page. + async fn left_by_the_filter( + middleware: &ClientHints, + services: &RuntimeServices, + request: &mut http::Request, + ) -> Vec { + left_in(&settings_with(PLACED), middleware, services, request).await + } + + /// What the filter leaves in `request`'s state for a deployment with + /// `settings`. + async fn left_in( + settings: &Settings, + middleware: &ClientHints, + services: &RuntimeServices, + request: &mut http::Request, + ) -> Vec { + let decision = middleware + .filter_request(RequestFilterInput { + settings, + services, + request, + geo_info: None, + permissions: None, + is_integration_route: false, + }) + .await + .expect("the filter refuses nothing"); + assert!(matches!(decision, RequestFilterDecision::Continue(_))); + IntegrationRequestState::of(request) + .get::(BUILDER_ID) + .map_or_else(Vec::new, |hints| hints.names().to_vec()) + } + + /// What the filter leaves for a first page view. + async fn prepared(middleware: &ClientHints, services: &RuntimeServices) -> Vec { + left_by_the_filter(middleware, services, &mut first_view()).await + } + + #[tokio::test] + async fn an_asset_request_asks_the_service_nothing_and_leaves_nothing() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + + let names = left_by_the_filter(&middleware(CLOUD), &services, &mut an_asset()).await; + + assert!(names.is_empty()); + assert!(client.sent().is_empty()); + } + + /// A first page view of `path`. + fn first_view_of(path: &str) -> http::Request { + let mut request = first_view(); + *request.uri_mut() = format!("https://www.example.com{path}") + .parse() + .expect("should build the address"); + request + } + + /// The table alone runs the filter, and with no entry naming the + /// middleware no page gets the headers, so no page asks the service. + #[tokio::test] + async fn a_table_with_no_entry_asks_the_service_nothing() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + + let names = left_in( + &settings_with(""), + &middleware(CLOUD), + &services, + &mut first_view(), + ) + .await; + + assert!(names.is_empty()); + assert!(client.sent().is_empty(), "no entry names the middleware"); + } + + /// An entry's `path` scopes the call as it scopes the headers, so a page + /// the entry covers asks the service and one it does not asks nothing. + #[tokio::test] + async fn an_entrys_path_scopes_the_call_to_the_service() { + let settings = settings_with(concat!( + "[[serve]]\n", + "media_type = \"text/html\"\n", + "path = \"/news/\"\n", + "middleware = [\"51degrees.client_hints\"]\n", + )); + for (path, covered) in [("/", false), ("/newsletter", false), ("/news/today", true)] { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + + let names = left_in( + &settings, + &middleware(CLOUD), + &services, + &mut first_view_of(path), + ) + .await; + + assert_eq!(client.sent().len(), usize::from(covered), "{path}"); + assert_eq!(names.is_empty(), !covered, "{path}"); + } + } + + /// The filter hands the service the reader's browser alone, so neither + /// the reader's address nor a cookie reaches the request sent. + #[tokio::test] + async fn the_filter_sends_no_address_and_no_cookie() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())) + .with_client_info(trusted_server_core::platform::ClientInfo { + client_ip: Some("2.125.160.216".parse().expect("should parse the address")), + ..Default::default() + }); + let mut request = first_view(); + request.headers_mut().insert( + http::header::COOKIE, + http::HeaderValue::from_static("51D_ScreenPixelsWidth=1080; ts-ec=an-id"), + ); + + let names = left_by_the_filter(&middleware(CLOUD), &services, &mut request).await; + + assert!(!names.is_empty()); + let sent = client.sent(); + assert_eq!(sent.len(), 1); + let body = &sent[0].body; + assert!( + body.contains("user-agent=") && body.contains("sec-ch-ua-mobile="), + "{body}" + ); + for absent in [ + "client-ip", + "2.125.160.216", + "51D_", + "1080", + "ts-ec", + "an-id", + ] { + assert!( + !body.contains(absent), + "{absent} reached the service: {body}" + ); + } + } + + #[tokio::test] + async fn a_first_view_is_given_the_services_list() { + // The case the tag exists for. The browser has been asked for nothing + // yet, and the list comes from the service, not from the request. + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client, Arc::new(MemoryKvStore::default())); + + let names = prepared(&middleware(CLOUD), &services).await; + + assert_eq!( + names, + vec![ + "sec-ch-ua".to_owned(), + "sec-ch-ua-platform".to_owned(), + "sec-ch-ua-model".to_owned(), + "sec-ch-ua-platform-version".to_owned(), + ] + ); + } + + #[tokio::test] + async fn the_held_answer_serves_the_next_request_without_a_call() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + let middleware = middleware(CLOUD); + + for _ in 0..2 { + assert!(!prepared(&middleware, &services).await.is_empty()); + } + + assert_eq!( + client.sent().len(), + 1, + "the answer, once, and the second page reads the held answer" + ); + } + + #[tokio::test] + async fn a_service_naming_nothing_leaves_nothing_to_delegate() { + // The data call answers an empty body, as a service that names no + // hints for this browser does. + let client = Arc::new(ScriptedHttpClient::new(&["{}"])); + let services = services_with(client, Arc::new(MemoryKvStore::default())); + + assert!(prepared(&middleware(CLOUD), &services).await.is_empty()); + } + + /// What the filter leaves is what the page carries, through the document + /// state core seeds from the request's state on serve. + #[tokio::test] + async fn what_the_filter_leaves_is_what_the_page_carries() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client, Arc::new(MemoryKvStore::default())); + let middleware = middleware(CLOUD); + let names = prepared(&middleware, &services).await; + let expected: Vec<&str> = names.iter().map(String::as_str).collect(); + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = serve_context(&state, &prepared, &expected); + + let inserts = middleware.create(&context).head_inserts; + + assert_eq!(inserts, delegation(CLOUD, &expected)); + assert_eq!(inserts.len(), 1); + } + + /// What the service answers for a browser engine that has no client + /// hints: `Unknown` for all three properties. + const DATA_NO_HINTS: &str = r#"{"device":{ + "setheaderbrowseraccept-ch":"Unknown", + "setheaderhardwareaccept-ch":"Unknown", + "setheaderplatformaccept-ch":"Unknown"}}"#; + + const GECKO: &str = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0"; + const WEBKIT: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 \ + (KHTML, like Gecko) Version/17.6 Safari/605.1.15"; + const CHROMIUM: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"; + + /// The page and the headers a first view from `user_agent` gets, when + /// the service answers `data`: the Delegate-CH inserts and the + /// `Accept-CH` header, each empty when the service names no hint. + async fn delegation_for_browser( + user_agent: &'static str, + data: &str, + ) -> (Vec, Option) { + let client = Arc::new(ScriptedHttpClient::new(&[data])); + let services = services_with(client, Arc::new(MemoryKvStore::default())); + let middleware = middleware(CLOUD); + let mut request = first_view_from(user_agent); + + let names = left_by_the_filter(&middleware, &services, &mut request).await; + + let names: Vec<&str> = names.iter().map(String::as_str).collect(); + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = serve_context(&state, &prepared, &names); + let inserts = middleware.create(&context).head_inserts; + let accept_ch = headers_for(&middleware, &names, &[]) + .into_iter() + .find(|(name, _)| name == "accept-ch") + .map(|(_, value)| value); + (inserts, accept_ch) + } + + /// The service is asked server side on the first request, before the + /// browser has sent a single hint, and a Chromium browser is given the + /// delegation and the headers from that answer. + #[tokio::test] + async fn a_chromium_browser_is_given_the_delegation_on_its_first_request() { + let (inserts, accept_ch) = delegation_for_browser(CHROMIUM, DATA).await; + + assert_eq!(inserts.len(), 1, "{inserts:?}"); + assert!( + inserts[0].contains(r#"http-equiv="Delegate-CH""#), + "{inserts:?}" + ); + assert_eq!( + accept_ch.as_deref(), + Some("sec-ch-ua, sec-ch-ua-platform, sec-ch-ua-model, sec-ch-ua-platform-version") + ); + } + + /// A browser engine with no client hints is never given the tag or the + /// headers, because the service answers `Unknown` for it and nothing + /// here keeps a list of its own. + #[tokio::test] + async fn a_gecko_browser_is_never_given_client_hint_headers() { + let (inserts, accept_ch) = delegation_for_browser(GECKO, DATA_NO_HINTS).await; + + assert!(inserts.is_empty(), "{inserts:?}"); + assert_eq!(accept_ch, None); + } + + #[tokio::test] + async fn a_webkit_browser_is_never_given_client_hint_headers() { + let (inserts, accept_ch) = delegation_for_browser(WEBKIT, DATA_NO_HINTS).await; + + assert!(inserts.is_empty(), "{inserts:?}"); + assert_eq!(accept_ch, None); + } + + // A browser that holds the hints. + + /// The names `DATA` gives. + const DATA_NAMES: &[&str] = &[ + "sec-ch-ua", + "sec-ch-ua-platform", + "sec-ch-ua-model", + "sec-ch-ua-platform-version", + ]; + + /// A Chromium page request carrying `hints`, as a browser does once a + /// response from the origin named them in `Accept-CH`. Each is the empty + /// string Chromium sends for `Sec-CH-UA-Model` on a desktop. + fn later_view(hints: &[&str]) -> http::Request { + let mut request = first_view_from(CHROMIUM); + for hint in hints { + request.headers_mut().insert( + http::HeaderName::from_bytes(hint.as_bytes()).expect("a hint is a header name"), + http::HeaderValue::from_static("\"\""), + ); + } + request + } + + /// Whether the filter marks `request` as carrying every name the service + /// answers with `DATA`. + async fn held_after_filter(mut request: http::Request) -> bool { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client, Arc::new(MemoryKvStore::default())); + + let names = left_by_the_filter(&middleware(CLOUD), &services, &mut request).await; + + assert!(!names.is_empty(), "the service names hints for Chromium"); + IntegrationRequestState::of(&request) + .get::(BUILDER_ID) + .is_some() + } + + #[tokio::test] + async fn a_request_carrying_every_hint_is_held() { + assert!(held_after_filter(later_view(DATA_NAMES)).await); + } + + #[tokio::test] + async fn a_request_missing_one_hint_is_not_held() { + // As when the service starts naming a hint, so the headers replace + // the set the browser holds. + assert!(!held_after_filter(later_view(&DATA_NAMES[..3])).await); + } + + #[tokio::test] + async fn the_hints_sent_unasked_do_not_count_as_held() { + // Chromium sends these three on every request, asked or not. + assert!(!held_after_filter(first_view()).await); + } + + #[test] + fn a_held_set_gets_the_policy_and_the_tag_without_accept_ch() { + let headers = headers_with( + &middleware(CLOUD), + &["sec-ch-ua", "sec-ch-ua-model"], + &[], + true, + ); + let state = IntegrationDocumentState::default(); + state.get_or_insert_with(BUILDER_ID, || HintsHeld); + let prepared = Prepared::default(); + let context = serve_context(&state, &prepared, &["sec-ch-ua-model"]); + + assert_eq!( + headers, + pairs(&[( + "permissions-policy", + r#"ch-ua=(self "https://cloud.example.com"), ch-ua-model=(self "https://cloud.example.com")"# + )]), + "the browser holds the set, and the policy applies to this page alone" + ); + assert_eq!( + middleware(CLOUD).create(&context).head_inserts.len(), + 1, + "the tag applies to this page alone" + ); + } + + #[test] + fn a_response_naming_its_own_set_still_gets_the_hints_added() { + // The response's set would replace the one the browser holds. + let headers = headers_with( + &middleware(CLOUD), + &["sec-ch-ua-model"], + &[("accept-ch", "Sec-CH-Prefers-Color-Scheme")], + true, + ); + + assert_eq!( + headers, + pairs(&[ + ("accept-ch", "Sec-CH-Prefers-Color-Scheme, sec-ch-ua-model"), + ("critical-ch", "sec-ch-ua-model"), + ( + "permissions-policy", + r#"ch-ua-model=(self "https://cloud.example.com")"# + ), + ]) + ); + } + + #[test] + fn a_hint_counts_as_carried_whatever_its_value() { + let mut headers = http::HeaderMap::new(); + headers.insert("sec-ch-ua-model", http::HeaderValue::from_static("\"\"")); + + assert!(carries_every(&headers, &["sec-ch-ua-model".to_owned()])); + assert!(!carries_every(&headers, &[]), "no names is never held"); + } + + // The tables. + + #[test] + fn critical_is_on_unless_the_table_turns_it_off() { + assert!(critical_from_settings(&settings_with("")).expect("no table is on")); + assert!( + critical_from_settings(&settings_with( + "[middleware.51degrees.client_hints]\ncritical = true\n" + )) + .expect("the table says on") + ); + assert!( + !critical_from_settings(&settings_with( + "[middleware.51degrees.client_hints]\ncritical = false\n" + )) + .expect("the table says off") + ); + } + + /// A misspelt key in the middleware's own table fails the deploy check + /// naming the table and the key, where it would otherwise be ignored. + #[test] + fn the_middlewares_table_refuses_a_key_it_does_not_know() { + let settings = + settings_with("[middleware.51degrees.client_hints]\nno_such_setting = true\n"); + + let message = format!( + "{:?}", + critical_from_settings(&settings) + .expect_err("the table carries a key it does not know") + ); + + assert!( + message.contains("[middleware.51degrees.client_hints]") + && message.contains("no_such_setting"), + "the message has to name the table and the key: {message}" + ); + assert!( + validate(&settings).is_err(), + "and the deploy check refuses it" + ); + } + + #[test] + fn a_configuration_without_the_shared_table_registers_nothing() { + let settings = page_settings(""); + + assert!( + register(&settings) + .expect("should read a document that selects nothing") + .is_none() + ); + assert!(!validate(&settings).expect("should read a document that selects nothing")); + } + + #[test] + fn a_shared_table_with_no_license_key_is_refused() { + let settings = page_settings(&format!("[shared.51degrees]\nendpoint = \"{CLOUD}\"\n")); + + let message = format!( + "{:?}", + register(&settings) + .err() + .expect("there is no key to present") + ); + + assert!( + message.contains("[shared.51degrees] has no `license_key`"), + "{message}" + ); + assert!(validate(&settings).is_err()); + } + + #[test] + fn the_shared_table_registers_the_filter_and_the_middleware() { + let registration = register(&settings_with("")) + .expect("should read the table") + .expect("the table runs the builder"); + + assert_eq!(registration.integration_id, BUILDER_ID); + assert!( + registration + .middleware + .iter() + .any(|middleware| middleware.middleware_id() == MODULE), + "the middleware should be offered for an entry to write" + ); + assert_eq!( + registration.request_filters.len(), + 1, + "the filter that asks the service" + ); + assert!( + registration.js_disabled, + "the headers and the tag are the whole client side" + ); + } + + #[test] + fn the_description_is_what_the_shared_table_registers() { + trusted_server_core::test_support::middleware::assert_described_middleware_is_registered( + &settings_with(""), + &linked_builders(), + BUILDER_ID, + ); + } + + /// The builders of the two 51Degrees modules, the client hints + /// middleware's and the loading cache's, as a deployment that wants the + /// answers held links them. + fn linked_builders() -> [IntegrationBuilder; 2] { + [builder(), trusted_server_loading_cache_51degrees::builder()] + } + + /// The middleware the serve phase's headers step asks for a page, which + /// is none where no entry names it, so a deployment that names no + /// `[[serve]]` entry sends no client hints headers. + #[test] + fn no_entry_means_no_headers() { + let unplaced = settings_with(""); + let registry = IntegrationRegistry::with_registrations(&unplaced, &linked_builders()) + .expect("should build the registry"); + assert!( + registry + .middleware_in(MiddlewarePhase::Serve) + .contains(&MODULE), + "the table runs the middleware whether or not an entry places it" + ); + assert!( + registry + .middleware_for_path(unplaced.phase_entries(MiddlewarePhase::Serve), "/") + .is_empty(), + "and the headers step asks nothing where no entry names it" + ); + + let placed = settings_with( + "[[serve]]\nmedia_type = \"text/html\"\nmiddleware = [\"51degrees.client_hints\"]\n", + ); + let registry = IntegrationRegistry::with_registrations(&placed, &linked_builders()) + .expect("should build the registry"); + let asked: Vec<&str> = registry + .middleware_for_path(placed.phase_entries(MiddlewarePhase::Serve), "/") + .iter() + .map(|middleware| middleware.middleware_id()) + .collect(); + + assert_eq!(asked, [MODULE]); + } + + /// One `[shared.51degrees]` table runs the middleware, and a linked + /// loading cache module holds the answers it asks for, in the cache this + /// builder declares, with no line in the document or with + /// `[cache.51degrees-answers]` naming the module. + #[test] + fn the_shared_table_runs_the_middleware_and_a_linked_cache_holds_its_answers() { + for (case, rest) in [ + ("no cache table", ""), + ( + "the cache table naming the module", + "[cache.51degrees-answers]\nmodule = \"51degrees\"\nstore_seconds = 600\n", + ), + ] { + let settings = settings_with(rest); + let registry = IntegrationRegistry::with_registrations(&settings, &linked_builders()) + .expect("should build the registry"); + let caches = trusted_server_core::integrations::loading_caches_from( + &settings, + &linked_builders(), + ) + .expect("the declared cache selects the one loading cache module"); + + assert!(registry.integration_runs(BUILDER_ID), "{case}"); + assert!( + caches.for_feature(CACHE).keeps_values(), + "{case}: the linked loading cache should hold the answers" + ); + } + } + + /// A `[cache.]` table for a cache nothing declares is refused, so + /// a misspelt table fails the deploy check and not the first request. + #[test] + fn a_misspelt_cache_table_is_refused_naming_the_declared_cache() { + let settings = settings_with("[cache.51degrees-answer]\nmodule = \"51degrees\"\n"); + + let message = format!( + "{:?}", + trusted_server_core::integrations::loading_caches_from(&settings, &linked_builders()) + .expect_err("nothing declares the misspelt cache") + ); + + assert!( + message.contains("[cache.51degrees-answer]") && message.contains(CACHE), + "{message}" + ); + } +} diff --git a/crates/shared/51degrees/Cargo.toml b/crates/shared/51degrees/Cargo.toml new file mode 100644 index 0000000000..245b00dd6a --- /dev/null +++ b/crates/shared/51degrees/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "trusted-server-shared-51degrees" +description = "What the 51Degrees modules share: the vendor's table, the one call to the cloud service and its held answer, the evidence the call sends and the cache the answer is held in." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[features] +# Exposes test_services to the other 51Degrees crates' tests, with a loading +# cache in their services so what one call holds is what the next one reads. +test-utils = ["dep:trusted-server-loading-cache-51degrees", "trusted-server-core/test-utils"] + +[dependencies] +trusted-server-core = { workspace = true } +trusted-server-loading-cache-51degrees = { workspace = true, optional = true } +async-trait = { workspace = true } +bytes = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +log = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +sha2 = { workspace = true } +url = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt", "macros"] } +trusted-server-core = { workspace = true, features = ["test-utils"] } +trusted-server-loading-cache-51degrees = { workspace = true } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. +[package.metadata.maintainers] +owner = "51Degrees" +status = "vendor owned" diff --git a/crates/shared/51degrees/README.md b/crates/shared/51degrees/README.md new file mode 100644 index 0000000000..c1170dfe15 --- /dev/null +++ b/crates/shared/51degrees/README.md @@ -0,0 +1,27 @@ +# What the 51Degrees modules share + +`trusted-server-shared-51degrees` holds what more than one 51Degrees module +needs, so each module's crate under `crates//51degrees` depends on this +one and on none of the others, and none of it is in core. + +- `config`: the vendor's table, `[shared.51degrees]`, being the cloud + endpoint, the license key and the timeout. Its presence runs the 51Degrees + builder. +- `license`: the license key every server side call presents, and the + refusal when the table has none. +- `client`: the one call to the cloud service for a request's evidence and + the answer it gives, loaded through the loading cache. Where a deployment + links a loading cache module the answer is held there, so a question many + requests ask at once is asked once. A stock build links none, so there + every call asks the service. +- `evidence`: the evidence a request gives the call, being the `User-Agent` + and the `Sec-CH-UA` hints alone. +- `engine_cache`: the cache the answers are held in, `51degrees-answers`, + selected and sized in `[cache.51degrees-answers]`. +- `hints_engine`: the client hints question, which asks the service which + hints it wants from a browser. +- `test_services`: runtime services for the 51Degrees crates' tests, behind + the `test-utils` feature. + +The modules that use it are documented in the +[51Degrees guide](../../../docs/guide/integrations/51degrees.md). diff --git a/crates/shared/51degrees/src/client.rs b/crates/shared/51degrees/src/client.rs new file mode 100644 index 0000000000..f5576df4aa --- /dev/null +++ b/crates/shared/51degrees/src/client.rs @@ -0,0 +1,888 @@ +//! The one call to the cloud service for a request's evidence, and the +//! answer it gives. +//! +//! The call is a form POST to the service's `json` endpoint, made over the +//! platform's HTTP client. It presents the license key, never the resource +//! key. A resource key is restricted to the domains it was created for, +//! checked against the `Referer` or `Origin` header, neither of which a +//! server to server call sends, so it answers `401`. A license key has no +//! such restriction and answers with only the properties the call names. +//! +//! Answers load through the loading cache in [`crate::engine_cache`], each +//! whole call being the load, so where a deployment links a loading cache +//! module a question many requests ask at once is asked once. A stock build +//! links none, so there every call asks the service. The evidence is the +//! `User-Agent` and the `Sec-CH-UA` hints alone, so one held answer serves +//! every reader whose browser sends the same values. Failures are not held. + +use std::time::Duration; + +use error_stack::{Report, ResultExt as _}; +use trusted_server_core::platform::{ + LoadError, LoadedValue, Loader, PlatformBackendSpec, PlatformError, PlatformHttpRequest, + RuntimeServices, +}; + +use crate::BACKEND_DISCRIMINATOR; +use crate::engine_cache::{AnswerCache, answer_key}; +use crate::hints_engine::engine_base; + +/// The evidence one cloud call sends, as the fields [`evidence_fields`] gives. +/// The call's key in the loading cache is built from the same fields the call +/// sends, so the key cannot drift from the request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Evidence { + /// The visitor's `User-Agent`. + pub user_agent: String, + /// The `Sec-CH-UA` hints the browser sent, by lower case header name and + /// in name order. Different hints are a different question, so they are + /// held as a different entry. + pub client_hints: Vec<(String, String)>, +} + +/// A decoded cloud response, the answer to one call's question. +#[derive(Debug, Clone)] +pub struct CloudAnswer { + body: serde_json::Value, +} + +impl CloudAnswer { + /// Wraps a decoded response. + #[must_use] + pub const fn new(body: serde_json::Value) -> Self { + Self { body } + } + + /// The whole decoded response. + #[must_use] + pub const fn body(&self) -> &serde_json::Value { + &self.body + } + + /// One element of the response, such as `device`. + /// + /// `None` when the credential is not entitled to that product, which is + /// ordinary, and the module reading it resolves nothing rather than + /// erroring. + #[must_use] + pub fn element(&self, name: &str) -> Option<&serde_json::Value> { + self.body.get(name) + } +} + +/// The form fields the call sends for `evidence`, under the bare header names +/// the service reads. +/// +/// No usage is sent, so no identifier comes back. An empty value is left out, +/// because sending one asks about an empty `User-Agent`. +#[must_use] +pub fn evidence_fields(evidence: &Evidence) -> Vec<(String, String)> { + let mut fields = Vec::new(); + if !evidence.user_agent.trim().is_empty() { + fields.push(("user-agent".to_owned(), evidence.user_agent.clone())); + } + for (name, value) in &evidence.client_hints { + if !value.is_empty() { + fields.push((name.clone(), value.clone())); + } + } + fields +} + +/// The file the service answers data requests at, under the endpoint's base. +const DATA_ENDPOINT: &str = "json"; + +/// Cap on the body read back from the service, so a wrong endpoint cannot grow +/// the instance's heap without bound. +const MAX_RESPONSE_BYTES: usize = 1024 * 1024; + +/// Calls the cloud service. The 51Degrees modules share one, and each +/// registers the properties it reads, so one call for a request carries them +/// all. +/// +/// Each answer loads through the loading cache, so where a deployment links a +/// loading cache module a question many requests ask at once is asked once. +#[derive(Debug, Clone)] +pub struct CloudPipeline { + /// The base the data endpoint is under, being the configured endpoint + /// without its last path segment, which is the segment a multi-tenant + /// endpoint carries the resource key in, so this call cannot present it. + base: String, + /// The license key the call authenticates with. Never a resource key. + license_key: String, + timeout: Duration, + /// Where the answers are held, and for which publisher. + cache: AnswerCache, + /// Every property a module of this deployment reads from an answer, + /// registered with [`wanting`](Self::wanting) when the deployment starts, + /// so one call carries them all. + properties: Vec<&'static str>, +} + +impl CloudPipeline { + /// Creates a pipeline for one endpoint and license key, holding its + /// answers in `cache` and asking for nothing until a module registers + /// what it reads. + /// + /// Returns `None` when the endpoint is not a URL, which startup validation + /// refuses anyway. There is no constructor taking a resource key, so a + /// server side call cannot present one. + #[must_use] + pub fn new( + endpoint: &str, + license_key: &str, + timeout_ms: u32, + cache: AnswerCache, + ) -> Option { + Some(Self { + base: engine_base(endpoint)?, + license_key: license_key.to_owned(), + timeout: Duration::from_millis(u64::from(timeout_ms)), + cache, + properties: Vec::new(), + }) + } + + /// Registers the properties a module reads from every answer, so the one + /// call a request makes carries them. Called for each module when the + /// deployment starts, and a property registered twice is asked for once. + #[must_use] + pub fn wanting(mut self, properties: &[&'static str]) -> Self { + for property in properties { + if !self.properties.contains(property) { + self.properties.push(property); + } + } + self + } + + /// Every property the registered modules read, in registration order. + #[must_use] + pub fn properties(&self) -> &[&'static str] { + &self.properties + } + + /// The answer for a request's evidence, held from an earlier call or + /// asked of the service. + /// + /// # Errors + /// + /// When the service cannot be reached, refuses the call, or answers with + /// something that is not JSON. + pub async fn answer( + &self, + evidence: &Evidence, + services: &RuntimeServices, + ) -> Result> { + let bytes = self + .call_bytes( + &self.properties, + evidence_fields(evidence), + crate::engine_cache::ANSWER_LIFETIME, + services, + ) + .await?; + Self::decode(&bytes) + } + + fn decode(bytes: &[u8]) -> Result> { + let body: serde_json::Value = serde_json::from_slice(bytes) + .change_context(PlatformError::HttpClient) + .attach("the held answer is not JSON")?; + Ok(CloudAnswer::new(body)) + } + + /// The bytes held for this question, or the service's answer, which is + /// then held for `lifetime`. + async fn call_bytes( + &self, + values: &[&str], + evidence: Vec<(String, String)>, + lifetime: Duration, + services: &RuntimeServices, + ) -> Result> { + let key = answer_key(&self.base, &self.license_key, values, &evidence); + let pipeline = self.clone(); + let values: Vec = values.iter().map(|value| (*value).to_owned()).collect(); + let call_services = services.clone(); + let load: Loader = Box::new(move || { + Box::pin(async move { + let json = pipeline + .ask_service(values, evidence, &call_services) + .await + .map_err(|report| LoadError::new(format!("{report:?}")))?; + Ok(LoadedValue::new(json.into_bytes()).with_lifetime(lifetime)) + }) + }); + self.cache + .held(services, key, load) + .await + .map_err(|error| Report::new(PlatformError::HttpClient).attach(error.to_string())) + } + + /// Sends one data request and returns the answer's JSON text. + /// + /// The license key and the property list lead the form, as the published + /// engines send them, and the evidence follows under the bare names the + /// service reads. + async fn ask_service( + &self, + values: Vec, + evidence: Vec<(String, String)>, + services: &RuntimeServices, + ) -> Result> { + let mut form = vec![ + ("license".to_owned(), self.license_key.clone()), + ("values".to_owned(), values.join(",")), + ]; + form.extend(evidence); + let outbound = http::Request::builder() + .method(http::Method::POST) + .uri(format!("{}{DATA_ENDPOINT}", self.base)) + .header(http::header::ACCEPT, "application/json") + .header( + http::header::CONTENT_TYPE, + "application/x-www-form-urlencoded", + ) + .body(edgezero_core::body::Body::from( + form_urlencoded_body(&form).into_bytes(), + )) + .change_context(PlatformError::HttpClient) + .attach("could not build the cloud request")?; + + let backend = services + .backend() + .ensure(&backend_spec(&self.base, self.timeout)?) + .change_context(PlatformError::HttpClient) + .attach("could not resolve a backend for the cloud endpoint")?; + let response = services + .http_client() + .send(PlatformHttpRequest::new(outbound, backend)) + .await + .change_context(PlatformError::HttpClient) + .attach("the cloud service did not answer")?; + + let status = response.response.status(); + let bytes = response + .response + .into_body() + .into_bytes_bounded(MAX_RESPONSE_BYTES) + .await + .map_err(|_| { + Report::new(PlatformError::HttpClient) + .attach("could not read the cloud service's answer") + })?; + let json = String::from_utf8_lossy(&bytes).into_owned(); + if !status.is_success() { + return Err(Report::new(PlatformError::HttpClient).attach(format!( + "the cloud service answered {status}: {}", + excerpt(&json) + ))); + } + // Checked before it is held, so what is held is always an answer. The + // service refuses a call with a list of errors and a success status, + // so that list is read as the refusal it is. + let body: serde_json::Value = serde_json::from_str(&json) + .change_context(PlatformError::HttpClient) + .attach("the cloud service did not return JSON")?; + if let Some(errors) = body.get("errors").and_then(serde_json::Value::as_array) + && !errors.is_empty() + { + return Err(Report::new(PlatformError::HttpClient).attach(format!( + "the cloud service refused the call: {}", + excerpt(&serde_json::Value::Array(errors.clone()).to_string()) + ))); + } + Ok(json) + } +} + +/// The start of `text`, for a message about an answer that was not one. +fn excerpt(text: &str) -> &str { + let end = text + .char_indices() + .nth(200) + .map_or(text.len(), |(index, _)| index); + &text[..end] +} + +/// The backend the data endpoint under `base` is reached through, with the +/// deployment's own timeout on the connection. +fn backend_spec( + base: &str, + timeout: Duration, +) -> Result> { + let url = url::Url::parse(base) + .change_context(PlatformError::HttpClient) + .attach("the cloud endpoint is not a URL")?; + let host = url + .host_str() + .ok_or_else(|| { + Report::new(PlatformError::HttpClient).attach("the cloud endpoint names no host") + })? + .to_owned(); + Ok(PlatformBackendSpec { + scheme: url.scheme().to_owned(), + host, + port: url.port(), + host_header_override: None, + certificate_check: true, + first_byte_timeout: timeout, + between_bytes_timeout: timeout, + discriminator: Some(BACKEND_DISCRIMINATOR.to_owned()), + }) +} + +/// The url-encoded form body the data endpoint expects. +fn form_urlencoded_body(form: &[(String, String)]) -> String { + let mut serializer = url::form_urlencoded::Serializer::new(String::new()); + for (name, value) in form { + serializer.append_pair(name, value); + } + serializer.finish() +} + +/// The properties in which the service names the hints it wants, one per +/// product, so the header follows the service. +const ACCEPT_CH_PROPERTIES: &[&str] = &[ + "setheaderbrowseraccept-ch", + "setheaderhardwareaccept-ch", + "setheaderplatformaccept-ch", +]; + +/// The `Accept-CH` value the service asked for, or `None` when it named none. +/// The three properties overlap, so their union is taken in order of first +/// appearance, which keeps the header stable between requests. +#[must_use] +pub fn accept_ch_from_answer(answer: &CloudAnswer) -> Option { + let device = answer.element("device")?; + let mut hints: Vec<&str> = Vec::new(); + for property in ACCEPT_CH_PROPERTIES { + let Some(value) = device.get(*property).and_then(serde_json::Value::as_str) else { + continue; + }; + for hint in value.split(',').map(str::trim) { + if hint.is_empty() || hint.eq_ignore_ascii_case(crate::UNKNOWN) { + continue; + } + if !hints.iter().any(|seen| seen.eq_ignore_ascii_case(hint)) { + hints.push(hint); + } + } + } + (!hints.is_empty()).then(|| hints.join(", ")) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + use crate::test_services::{ + MemoryKvStore, MemoryValueCache, ScriptedAnswer, ScriptedHttpClient, services_with, + services_with_value_cache, + }; + + /// What the data endpoint answers. + const DATA: &str = r#"{"device":{"ismobile":false,"browsername":"Chrome"}}"#; + + const PROPERTIES: &[&str] = &["device.IsMobile", "device.BrowserName"]; + + fn evidence() -> Evidence { + Evidence { + user_agent: "Mozilla/5.0".to_owned(), + client_hints: Vec::new(), + } + } + + fn pipeline() -> CloudPipeline { + CloudPipeline::new( + "https://cloud.example.com/api/v4/a-public-resource-key.json", + "a-secret-license-key", + 500, + crate::test_services::answer_cache(), + ) + .expect("should build") + .wanting(PROPERTIES) + } + + fn browser_name(answer: &CloudAnswer) -> Option<&str> { + answer + .element("device") + .and_then(|device| device["browsername"].as_str()) + } + + // The credential. + + /// A server side call presents the license key and never the public + /// resource key, which answers `401` to a call that sends no `Referer` + /// or `Origin`. + #[tokio::test] + async fn the_call_presents_the_license_key_and_never_the_resource_key() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + + let answer = pipeline() + .answer(&evidence(), &services) + .await + .expect("the call should succeed"); + + let sent = client.sent(); + let data = sent.last().expect("the data request should have been sent"); + assert!( + data.body.contains("license=a-secret-license-key"), + "got {}", + data.body + ); + assert!( + !data.body.contains("resource="), + "a resource key is public by design and a call the browser never sees has no \ + business spending it: {}", + data.body + ); + assert!( + !sent.iter().any(|request| { + request.uri.contains("a-public-resource-key") + || request.body.contains("a-public-resource-key") + }), + "and the key in the configured endpoint path must not reach the address or the \ + body either: {sent:?}" + ); + assert_eq!(browser_name(&answer), Some("Chrome")); + } + + // What is asked, and where. + + /// The data request is a form POST to the `json` file under the + /// endpoint's base, as the published engines send it. + #[tokio::test] + async fn the_call_is_a_form_post_to_the_data_endpoint() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + + pipeline() + .answer(&evidence(), &services) + .await + .expect("the call should succeed"); + + let sent = client.sent(); + assert_eq!(sent.len(), 1, "one data call and no discovery"); + assert_eq!(sent[0].uri, "https://cloud.example.com/api/v4/json"); + assert_eq!(sent[0].method, "POST"); + assert!( + sent[0].headers.iter().any(|(name, value)| { + name == "content-type" && value == "application/x-www-form-urlencoded" + }), + "{:?}", + sent[0].headers + ); + } + + /// The `User-Agent` and each hint reach the service under the bare + /// header names it reads. + #[tokio::test] + async fn the_evidence_reaches_the_service_under_the_bare_names_it_reads() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + let with_hint = Evidence { + client_hints: vec![("sec-ch-ua-mobile".to_owned(), "?0".to_owned())], + ..evidence() + }; + + pipeline() + .answer(&with_hint, &services) + .await + .expect("the call should succeed"); + + let sent = client.sent(); + let body = &sent.last().expect("the data request").body; + assert!( + body.contains("user-agent=Mozilla%2F5.0") && body.contains("sec-ch-ua-mobile=%3F0"), + "the bare evidence names are what the service reads, got {body}" + ); + } + + /// An empty value is not evidence, and sending one asks the service to + /// describe an empty `User-Agent`. + #[test] + fn an_empty_value_is_left_out_of_the_evidence() { + let hints_only = Evidence { + user_agent: String::new(), + client_hints: vec![ + ("sec-ch-ua-mobile".to_owned(), "?0".to_owned()), + ("sec-ch-ua-model".to_owned(), String::new()), + ], + }; + + assert_eq!( + evidence_fields(&hints_only), + vec![("sec-ch-ua-mobile".to_owned(), "?0".to_owned())] + ); + } + + /// A server side call never asks for an identifier, whatever the visitor + /// has answered, because an identifier is created only in the browser. + #[test] + fn a_server_side_call_never_asks_for_an_identifier() { + assert!( + evidence_fields(&evidence()) + .iter() + .all(|(key, _)| !key.ends_with("usage")), + "no usage may reach a server side call", + ); + } + + /// A license key names nothing of its own, so the call names every + /// property the modules registered, and nothing else comes back. + #[tokio::test] + async fn a_server_side_call_names_every_property_it_wants() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + + pipeline() + .answer(&evidence(), &services) + .await + .expect("the call should succeed"); + + let sent = client.sent(); + let body = &sent.last().expect("the data request").body; + let asked: Vec = body + .split('&') + .filter_map(|field| field.strip_prefix("values=")) + .flat_map(|list| list.split("%2C")) + .map(str::to_owned) + .collect(); + + assert_eq!( + asked, + PROPERTIES + .iter() + .map(|value| (*value).to_owned()) + .collect::>(), + "a license key names nothing of its own, so a call that names nothing gets \ + nothing: {body}" + ); + } + + #[test] + fn a_property_registered_twice_is_asked_for_once() { + let pipeline = pipeline().wanting(&["device.IsMobile", "device.HardwareName"]); + + assert_eq!( + pipeline.properties(), + &[ + "device.IsMobile", + "device.BrowserName", + "device.HardwareName" + ] + ); + } + + // The cache, counted. + + /// The cache reduces calls. Two instances share only the key-value store, + /// which holds the answers on a host with no value cache, and the second + /// asks the service nothing. + #[tokio::test] + async fn a_second_instance_with_the_same_evidence_asks_the_cloud_nothing() { + let store = Arc::new(MemoryKvStore::default()); + let first = Arc::new(ScriptedHttpClient::new(&[DATA])); + pipeline() + .answer(&evidence(), &services_with(first.clone(), store.clone())) + .await + .expect("the first call should succeed"); + assert_eq!( + first.sent().len(), + 1, + "the first instance pays for the call" + ); + + let second = Arc::new(ScriptedHttpClient::new(&[DATA])); + let answer = pipeline() + .answer(&evidence(), &services_with(second.clone(), store)) + .await + .expect("the second call should succeed"); + + assert_eq!( + second.sent().len(), + 0, + "the answer was paid for by the first instance, so a second one with the same \ + evidence has nothing to ask" + ); + assert_eq!( + browser_name(&answer), + Some("Chrome"), + "and the answer still comes back" + ); + } + + /// With a host value cache and no key-value store, a second lookup with + /// the same evidence asks the cloud nothing. + #[tokio::test] + async fn the_host_value_cache_answers_a_second_lookup_without_the_store() { + let cache = Arc::new(MemoryValueCache::default()); + let first = Arc::new(ScriptedHttpClient::new(&[DATA])); + pipeline() + .answer( + &evidence(), + &services_with_value_cache(first.clone(), cache.clone()), + ) + .await + .expect("the first lookup should succeed"); + assert_eq!(first.sent().len(), 1, "the first lookup pays for the call"); + assert_eq!(cache.len(), 1, "the answer is held"); + + let second = Arc::new(ScriptedHttpClient::new(&[DATA])); + let answer = pipeline() + .answer( + &evidence(), + &services_with_value_cache(second.clone(), cache), + ) + .await + .expect("the second lookup should succeed"); + + assert_eq!( + second.sent().len(), + 0, + "the second lookup with the same evidence is answered from the value cache" + ); + assert_eq!(browser_name(&answer), Some("Chrome")); + } + + /// Evidence that differs from a question already answered is a question of + /// its own, so it costs a call. + #[tokio::test] + async fn different_evidence_still_costs_a_call() { + for (case, changed, why) in [ + ( + "a different User-Agent", + Evidence { + user_agent: "curl/8".to_owned(), + ..evidence() + }, + "the answer depends on the User-Agent", + ), + ( + "a hint the browser sent", + Evidence { + client_hints: vec![("sec-ch-ua-mobile".to_owned(), "?1".to_owned())], + ..evidence() + }, + "a hint changes the question", + ), + ] { + let store = Arc::new(MemoryKvStore::default()); + let first = Arc::new(ScriptedHttpClient::new(&[DATA])); + pipeline() + .answer(&evidence(), &services_with(first, store.clone())) + .await + .expect("the first call should succeed"); + + let second = Arc::new(ScriptedHttpClient::new(&[DATA])); + pipeline() + .answer(&changed, &services_with(second.clone(), store)) + .await + .expect("the second call should succeed"); + + assert_eq!( + second.sent().len(), + 1, + "{case} costs one call, because {why}" + ); + } + } + + /// Two requests asking the same question at the same moment, on two + /// threads, make one call to the service between them, and both get the + /// answer. The first asks while the second waits for it. + #[test] + fn two_requests_asking_at_once_make_one_call_between_them() { + let client = + Arc::new(ScriptedHttpClient::new(&[DATA]).with_delay(Duration::from_millis(200))); + // One instance, so one loading cache, as a native server's requests + // share. + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + let start = std::sync::Barrier::new(2); + + let names: Vec> = std::thread::scope(|scope| { + let handles: Vec<_> = (0..2) + .map(|_| { + scope.spawn(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("should build a runtime"); + start.wait(); + let answer = runtime + .block_on(pipeline().answer(&evidence(), &services)) + .expect("each request should be answered"); + browser_name(&answer).map(str::to_owned) + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().expect("a request should not panic")) + .collect() + }); + + assert_eq!(client.sent().len(), 1, "one call between both requests"); + assert_eq!(names, vec![Some("Chrome".to_owned()); 2]); + } + + /// An answer is held in the store for an hour, so the store is told the + /// time left, which is a moment less by the time the answer is written. + #[tokio::test] + async fn each_answer_is_held_for_its_own_lifetime() { + const HOUR: Duration = Duration::from_secs(60 * 60); + let store = Arc::new(MemoryKvStore::default()); + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + + pipeline() + .answer(&evidence(), &services_with(client, store.clone())) + .await + .expect("the call should succeed"); + + let prefix = format!( + "ts.publisher.example/{}/answer-", + crate::engine_cache::CACHE + ); + let held = store + .keys() + .into_iter() + .find(|key| key.starts_with(&prefix)) + .and_then(|key| store.lifetime(&key)); + assert!( + held.is_some_and(|held| held <= HOUR && HOUR - held < Duration::from_secs(1)), + "{held:?}, among {:?}", + store.keys() + ); + } + + // Failure. + + /// A failed lookup stays a failure, and nothing is held for the next + /// request to read back. + #[tokio::test] + async fn a_service_that_answers_nonsense_is_an_error_and_not_an_empty_answer() { + let client = Arc::new(ScriptedHttpClient::new(&["not json at all"])); + let services = services_with(client, Arc::new(MemoryKvStore::default())); + + assert!( + pipeline().answer(&evidence(), &services).await.is_err(), + "a failed lookup is not the same answer as a visitor the service knows nothing of" + ); + } + + /// The service refuses a call it cannot serve with a status, or with a + /// list of errors and a success status, and either is a failure that + /// names what the service said rather than an answer. + #[tokio::test] + async fn a_refusal_is_an_error_naming_what_the_service_said() { + for (case, answer, said) in [ + ( + "a status", + ScriptedAnswer::new(400, r#"{"errors":["license key not valid"]}"#), + "answered 400", + ), + ( + "a list of errors with a success status", + ScriptedAnswer::new(200, r#"{"errors":["license key not valid"]}"#), + "refused the call", + ), + ] { + let client = Arc::new(ScriptedHttpClient::answering([answer])); + let services = services_with(client, Arc::new(MemoryKvStore::default())); + + let message = format!( + "{:?}", + pipeline() + .answer(&evidence(), &services) + .await + .expect_err(case) + ); + + assert!( + message.contains(said) && message.contains("license key not valid"), + "{case}: {message}" + ); + } + } + + #[test] + fn an_endpoint_that_is_not_a_url_yields_no_pipeline() { + assert!( + CloudPipeline::new( + "not-a-url", + "license", + 500, + crate::test_services::answer_cache() + ) + .is_none(), + "startup validation refuses one anyway, so this is the belt" + ); + } + + #[test] + fn the_form_body_is_url_encoded() { + let body = form_urlencoded_body(&[ + ("license".to_owned(), "a key".to_owned()), + ( + "values".to_owned(), + "device.SetHeaderBrowserAccept-CH".to_owned(), + ), + ]); + + assert_eq!( + body, "license=a+key&values=device.SetHeaderBrowserAccept-CH", + "the data endpoint reads a form body, and an unencoded space ends the field" + ); + } + + // What the service asks the browser for. + + /// The `Accept-CH` value is the union of the three properties in order of + /// first appearance, so the header is stable between requests and + /// comparable in a log. `Unknown` is how the service spells absent, so it + /// is never asked for, and an answer naming no hints gives no value, + /// because an empty header would tell a browser to stop sending the hints + /// it already sends. + #[test] + fn the_accept_ch_value_is_the_union_of_the_hints_the_service_named() { + for (case, body, expected) in [ + ( + "three overlapping properties, as the live service answers", + serde_json::json!({"device": { + "setheaderbrowseraccept-ch": + "Sec-CH-UA,Sec-CH-UA-Full-Version-List,Sec-CH-UA-Mobile,Sec-CH-UA-Platform", + "setheaderhardwareaccept-ch": "Sec-CH-UA-Model,Sec-CH-UA-Mobile", + "setheaderplatformaccept-ch": "Sec-CH-UA-Platform,Sec-CH-UA-Platform-Version", + }}), + Some( + "Sec-CH-UA, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Mobile, \ + Sec-CH-UA-Platform, Sec-CH-UA-Model, Sec-CH-UA-Platform-Version", + ), + ), + ( + "one property spelled Unknown", + serde_json::json!({"device": { + "setheaderbrowseraccept-ch": "Unknown", + "setheaderhardwareaccept-ch": "Sec-CH-UA-Model", + }}), + Some("Sec-CH-UA-Model"), + ), + ( + "a device section naming no hints", + serde_json::json!({"device": {}}), + None, + ), + ("no device section", serde_json::json!({}), None), + ] { + assert_eq!( + accept_ch_from_answer(&CloudAnswer::new(body)).as_deref(), + expected, + "{case}" + ); + } + } +} diff --git a/crates/shared/51degrees/src/config.rs b/crates/shared/51degrees/src/config.rs new file mode 100644 index 0000000000..c1ce09bc54 --- /dev/null +++ b/crates/shared/51degrees/src/config.rs @@ -0,0 +1,178 @@ +//! The settings every 51Degrees module shares, from `[shared.51degrees]`. +//! +//! The table's presence is what runs the 51Degrees modules, so a configuration +//! without it runs none of them, and each module's own settings stay in the +//! table that selects it. The cache the modules' answers are held in is +//! selected and sized in `[cache.51degrees-answers]`, as every declared cache +//! is, see [`crate::engine_cache`]. + +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::redacted::Redacted; +use trusted_server_core::settings::{SHARED_TYPE, Settings}; +use validator::Validate; + +/// This vendor's table under `[shared]`, `51degrees`, taken from this crate's +/// folder. +#[must_use] +pub fn vendor() -> &'static str { + trusted_server_core::module_name::short_form(SHARED_TYPE, trusted_server_core::module_name!()) +} + +/// `[shared.51degrees]`, being the cloud endpoint, the license key and the +/// timeout the 51Degrees modules share. +#[derive(Debug, Clone, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct SharedConfig { + /// The full URL of the JSON endpoint, including any resource key the + /// deployment requires in the path, for example + /// `https://cloud.example.com/api/v4/.json`, or + /// `http://127.0.0.1:8080/api/v4/json` for a self-hosted container, which + /// carries no key. + /// + /// Server side calls take the last path segment off this endpoint, which is + /// the segment a multi-tenant service carries the resource key in, and + /// present [`license_key`](Self::license_key) instead. The origin of the + /// endpoint is where the client hints middleware delegates the browser's + /// hints to. + /// + /// Name the domains a resource key may be used from when creating it in the + /// 51Degrees configurator. A key with no domain list works from anywhere, so + /// anyone who reads it can spend the allowance against the account. + #[validate(url)] + pub endpoint: String, + + /// How long to wait for the service before giving up, in milliseconds. + /// The call sits in front of the page on a first view, so the default is + /// short. + #[serde(default = "default_timeout_ms")] + #[validate(range(min = 1, max = 10_000))] + pub timeout_ms: u32, + + /// The license key every server side call authenticates with. + /// + /// A server side call presents the license key and never the resource key. A + /// resource key is public, travelling to the browser inside a script URL, and + /// a key restricted to named domains answers `401` to a server, which sends + /// no `Referer` or `Origin`. A resource key also answers with everything it + /// carries whatever the call names. + /// + /// A secret, named in the settings by its entry in the secret store. Required + /// whenever the table is present, because every 51Degrees module presents + /// it, see [`license_key`](crate::license::license_key). + #[serde(default)] + pub license_key: Option>, +} + +const fn default_timeout_ms() -> u32 { + 500 +} + +/// Reads `[shared.51degrees]`, or `None` when the configuration has none. +/// +/// # Errors +/// +/// When the table carries a setting it does not know, or one that fails +/// validation. +pub fn read(settings: &Settings) -> Result, Report> { + settings.shared_config::(vendor()) +} + +#[cfg(test)] +mod tests { + use super::*; + use trusted_server_core::html_processor::test_support::page_settings; + + #[test] + fn the_table_is_named_by_this_crates_folder() { + assert_eq!(vendor(), "51degrees"); + } + + #[test] + fn a_configuration_without_the_table_reads_as_none() { + assert!( + read(&page_settings("")) + .expect("should read settings with no table") + .is_none() + ); + } + + #[test] + fn the_table_is_read_with_its_defaults() { + let settings = page_settings( + r#" + [shared.51degrees] + endpoint = "https://cloud.example.com/api/v4/json" + license_key = "a-license-key" + "#, + ); + + let shared = read(&settings) + .expect("should read the table") + .expect("the table is present"); + + assert_eq!(shared.endpoint, "https://cloud.example.com/api/v4/json"); + assert_eq!(shared.timeout_ms, 500); + assert_eq!( + shared.license_key.as_ref().map(|key| key.expose().as_str()), + Some("a-license-key") + ); + } + + /// A misspelt key fails the deploy check naming the table and the key, + /// where it would otherwise be ignored. + #[test] + fn a_key_the_table_does_not_know_is_refused() { + let settings = page_settings( + r#" + [shared.51degrees] + endpoint = "https://cloud.example.com/api/v4/json" + license_key = "a-license-key" + no_such_setting = true + "#, + ); + + let message = format!( + "{:?}", + read(&settings).expect_err("the table carries a key it does not know") + ); + + assert!( + message.contains("[shared.51degrees]") && message.contains("no_such_setting"), + "the message has to name the table and the key: {message}" + ); + } + + #[test] + fn a_timeout_out_of_range_is_refused() { + let settings = page_settings( + r#" + [shared.51degrees] + endpoint = "https://cloud.example.com/api/v4/json" + license_key = "a-license-key" + timeout_ms = 0 + "#, + ); + + let message = format!( + "{:?}", + read(&settings).expect_err("a zero timeout waits for nothing") + ); + + assert!(message.contains("[shared.51degrees]"), "{message}"); + } + + #[test] + fn an_endpoint_that_is_not_a_url_is_refused() { + let settings = page_settings( + r#" + [shared.51degrees] + endpoint = "not a url" + license_key = "a-license-key" + "#, + ); + + assert!(read(&settings).is_err()); + } +} diff --git a/crates/shared/51degrees/src/engine_cache.rs b/crates/shared/51degrees/src/engine_cache.rs new file mode 100644 index 0000000000..9f3f0406c2 --- /dev/null +++ b/crates/shared/51degrees/src/engine_cache.rs @@ -0,0 +1,294 @@ +//! The cloud service's answers, loaded through the loading cache core +//! declares. +//! +//! Where a deployment links a loading cache module, each answer is held in +//! the host's shared store, which outlives an instance that serves one +//! request at a time, so the answer serves the next equivalent request and +//! any second lookup in the same request. A stock build links none, so there +//! every call asks the service. +//! +//! Each whole call is the loader, so where a loading cache module is linked a +//! question many requests ask at the same moment is asked of the service once, +//! and every request waiting on it gets the answer or the failure. A failure +//! is not held. The key is a hash of the endpoint, the license key, the +//! properties named and the evidence sent, so an answer is only ever given +//! back to the same question, and the license key is never written into a +//! store. +//! +//! The evidence is the `User-Agent` and the `Sec-CH-UA` hints, with no address +//! and no cookie, so one entry serves every reader whose browser sends the +//! same values. +//! +//! The cache is [`CACHE`], which the 51Degrees builder declares, and a +//! deployment selects the loading cache that holds it and sizes its copies in +//! `[cache.51degrees-answers]`, as for every declared cache. + +use std::time::Duration; + +use bytes::Bytes; +use sha2::{Digest as _, Sha256}; +use trusted_server_core::platform::{ + CacheLifetimes, CacheSpec, LoadError, Loader, RuntimeServices, StoreKey, +}; +use trusted_server_core::settings::{LoadingCacheConfig, Settings}; + +/// The cache the cloud service's answers are held in, and the name of its +/// `[cache.]` table. +pub const CACHE: &str = "51degrees-answers"; + +/// How long an answer is held, an hour, after which the service is asked +/// again. +pub(crate) const ANSWER_LIFETIME: Duration = Duration::from_secs(60 * 60); + +/// What a key's hash covers, so a change to it does not read old entries. +const KEY_FORMAT: u32 = 1; + +/// The lifetimes `[cache.51degrees-answers]` leaves unset. Each answer's own +/// lifetime ends it sooner than the store lifetime. +const DEFAULT_LIFETIMES: CacheLifetimes = CacheLifetimes { + memory: Duration::from_secs(60), + store: Duration::from_secs(7 * 24 * 60 * 60), + store_idle: Duration::from_secs(24 * 60 * 60), +}; + +/// One deployment's cache of cloud answers, with the lifetimes +/// `[cache.51degrees-answers]` sets. Each answer is held under the prefix of +/// the publisher the request's services carry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AnswerCache { + spec: CacheSpec, +} + +impl AnswerCache { + /// The cache with the lifetimes `config` sets. + #[must_use] + pub fn new(config: &LoadingCacheConfig) -> Self { + Self { + spec: CacheSpec { + name: CACHE, + lifetimes: config.lifetimes(DEFAULT_LIFETIMES), + memory_entries: 4096, + }, + } + } + + /// The cache with the lifetimes the settings' `[cache.51degrees-answers]` + /// sets, each its default where the table leaves it unset or there is no + /// table. + #[must_use] + pub fn for_settings(settings: &Settings) -> Self { + let config = settings + .cache + .declared + .get(CACHE) + .cloned() + .unwrap_or_default(); + Self::new(&config) + } + + /// The store key `key` is held under, for the publisher the request's + /// services carry. + pub(crate) fn store_key(services: &RuntimeServices, key: String) -> StoreKey { + StoreKey::new(services.server_key_prefix(), CACHE, key) + } + + /// The bytes held for `key`, or those `load` gives. + pub(crate) async fn held( + &self, + services: &RuntimeServices, + key: String, + load: Loader, + ) -> Result { + services + .load_cached(&self.spec, &Self::store_key(services, key), load) + .await + } +} + +/// Hashes the parts of a key, each length-prefixed so no two lists of parts +/// share a hash, into the text a store key ends with. +fn hashed(kind: &str, parts: &[&str]) -> String { + let mut hasher = Sha256::new(); + hasher.update(KEY_FORMAT.to_be_bytes()); + for part in std::iter::once(kind).chain(parts.iter().copied()) { + hasher.update((part.len() as u64).to_be_bytes()); + hasher.update(part.as_bytes()); + } + let mut key = format!("{kind}-"); + for byte in hasher.finalize() { + key.push_str(&format!("{byte:02x}")); + } + key +} + +/// The key one question is held under: the endpoint's base, the credential, +/// the properties named and the evidence sent, in any order. +pub(crate) fn answer_key( + base: &str, + license_key: &str, + values: &[&str], + evidence: &[(String, String)], +) -> String { + let mut values: Vec<&str> = values.to_vec(); + values.sort_unstable(); + values.dedup(); + let mut evidence: Vec<&(String, String)> = evidence.iter().collect(); + evidence.sort(); + let mut parts = vec![base, license_key, "values"]; + parts.extend(values); + parts.push("evidence"); + for (name, value) in evidence { + parts.push(name); + parts.push(value); + } + hashed("answer", &parts) +} + +#[cfg(test)] +mod tests { + use super::*; + + const BASE: &str = "https://cloud.example.com/api/v4/"; + + fn evidence(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect() + } + + #[test] + fn the_credential_never_reaches_the_store_key() { + let key = answer_key( + BASE, + "a-real-secret", + &["device.X"], + &evidence(&[("user-agent", "Mozilla")]), + ); + + assert!(!key.contains("a-real-secret"), "{key}"); + assert!(key.starts_with("answer-"), "{key}"); + } + + #[test] + fn a_store_key_fits_the_platform_limit() { + // A long evidence set still has to produce a key the store accepts. + let long = "Mozilla/5.0 ".repeat(400); + let key = answer_key( + BASE, + "k", + &["device.X"], + &evidence(&[("user-agent", &long)]), + ); + let stored = StoreKey::new( + &trusted_server_core::platform::ServerKeyPrefix::new( + "ts", + "a-long-publisher-name.example", + ), + CACHE, + key, + ); + + assert!(stored.to_string().len() <= 512, "{stored}"); + } + + /// Every answer is held under the prefix of the publisher the request's + /// services carry. + #[test] + fn an_answer_is_held_under_the_publishers_prefix() { + trusted_server_core::prefix::test_support::assert_keys_follow_the_publisher(|domain| { + let services = crate::test_services::services_with( + std::sync::Arc::new(crate::test_services::ScriptedHttpClient::new(&[])), + std::sync::Arc::default(), + ) + .with_prefixes(trusted_server_core::prefix::test_support::trusted_server_for(domain)); + AnswerCache::store_key(&services, answer_key(BASE, "k", &["device.X"], &[])).to_string() + }); + } + + #[test] + fn different_questions_do_not_share_an_entry() { + let one = answer_key(BASE, "k", &["device.X"], &evidence(&[("ua", "A")])); + + assert_ne!( + one, + answer_key(BASE, "k", &["device.X"], &evidence(&[("ua", "B")])), + "different evidence is a different entry" + ); + assert_ne!( + one, + answer_key(BASE, "k", &["device.Y"], &evidence(&[("ua", "A")])), + "a different property list is a different entry" + ); + assert_ne!( + one, + answer_key(BASE, "other", &["device.X"], &evidence(&[("ua", "A")])), + "another credential is a different entry" + ); + assert_ne!( + answer_key(BASE, "k", &["ab", "c"], &[]), + answer_key(BASE, "k", &["a", "bc"], &[]), + "parts that run together are still told apart" + ); + } + + #[test] + fn the_same_question_is_the_same_entry_in_any_order() { + assert_eq!( + answer_key( + BASE, + "k", + &["device.X", "ip.Y"], + &evidence(&[("a", "1"), ("b", "2")]) + ), + answer_key( + BASE, + "k", + &["ip.Y", "device.X"], + &evidence(&[("b", "2"), ("a", "1")]) + ), + ); + } + + #[test] + fn the_cache_takes_its_lifetimes_from_the_settings() { + let cache = AnswerCache::new(&LoadingCacheConfig { + module: None, + memory_seconds: Some(5), + store_seconds: None, + store_idle_seconds: Some(120), + }); + + assert_eq!(cache.spec.name, CACHE); + assert_eq!( + cache.spec.lifetimes, + CacheLifetimes { + memory: Duration::from_secs(5), + store: DEFAULT_LIFETIMES.store, + store_idle: Duration::from_secs(120), + } + ); + } + + /// The lifetimes come from the cache's own `[cache.51degrees-answers]` + /// table, as every declared cache's do, and are the defaults without it. + #[test] + fn the_cache_is_sized_by_its_own_table() { + use trusted_server_core::html_processor::test_support::page_settings; + + let sized = AnswerCache::for_settings(&page_settings(&format!( + "[cache.{CACHE}]\nmemory_seconds = 7\nstore_seconds = 3600\n" + ))); + let defaulted = AnswerCache::for_settings(&page_settings("")); + + assert_eq!( + sized.spec.lifetimes, + CacheLifetimes { + memory: Duration::from_secs(7), + store: Duration::from_secs(3600), + store_idle: DEFAULT_LIFETIMES.store_idle, + } + ); + assert_eq!(defaulted.spec.lifetimes, DEFAULT_LIFETIMES); + } +} diff --git a/crates/shared/51degrees/src/evidence.rs b/crates/shared/51degrees/src/evidence.rs new file mode 100644 index 0000000000..39c3743f99 --- /dev/null +++ b/crates/shared/51degrees/src/evidence.rs @@ -0,0 +1,101 @@ +//! The evidence a cloud call sends for a request, being the `User-Agent` and +//! the `Sec-CH-UA` client hints the browser sent with it. +//! +//! Which hints the service asks for depends on the browser alone, so nothing +//! else is sent, neither the address nor a cookie. An answer is keyed by the +//! evidence, so where a loading cache module holds it, every reader whose +//! browser sends the same values shares one answer and one call. + +use trusted_server_core::evidence::RequestInfo; + +use crate::client::Evidence; + +/// Builds the evidence for a request, being its `User-Agent` and each of its +/// `Sec-CH-UA` headers, by lower case name and in name order, so the same +/// headers always give the same cache key. +#[must_use] +pub fn evidence_for(request_info: &dyn RequestInfo) -> Evidence { + let mut client_hints: Vec<(String, String)> = request_info + .header_names() + .into_iter() + .filter(|name| is_user_agent_hint(name)) + .filter_map(|name| { + request_info + .header(name) + .map(|value| (name.to_ascii_lowercase(), value.to_owned())) + }) + .collect(); + client_hints.sort(); + Evidence { + user_agent: request_info.user_agent().to_owned(), + client_hints, + } +} + +/// Whether `name` is `Sec-CH-UA` or one of the `Sec-CH-UA-` hints, which +/// describe the browser and the device it runs on. +fn is_user_agent_hint(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + name == "sec-ch-ua" || name.starts_with("sec-ch-ua-") +} + +#[cfg(test)] +mod tests { + use trusted_server_core::evidence::OwnedRequestInfo; + + use super::*; + + const CHROME: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"; + + /// A request from an address, carrying `headers`. + fn request_with(headers: &[(&'static str, &'static str)]) -> OwnedRequestInfo { + let mut map = http::HeaderMap::new(); + for (name, value) in headers { + map.append( + http::HeaderName::from_static(name), + http::HeaderValue::from_static(value), + ); + } + OwnedRequestInfo::new("2.125.160.216".to_owned(), map) + } + + /// The `User-Agent` and the `Sec-CH-UA` hints are the whole of the + /// evidence. The address, the cookies, a hint outside that family and + /// every other header are left out. + #[test] + fn the_evidence_is_the_user_agent_and_the_sec_ch_ua_hints() { + let request = request_with(&[ + ("user-agent", CHROME), + ("sec-ch-ua-platform", "\"Windows\""), + ("sec-ch-ua", "\"Chromium\";v=\"140\""), + ("sec-ch-ua-mobile", "?0"), + ("sec-ch-prefers-color-scheme", "dark"), + ("cookie", "51D_ScreenPixelsWidth=1080; ts-ec=something"), + ("accept-language", "en-GB"), + ]); + + assert_eq!( + evidence_for(&request), + Evidence { + user_agent: CHROME.to_owned(), + client_hints: vec![ + ("sec-ch-ua".to_owned(), "\"Chromium\";v=\"140\"".to_owned()), + ("sec-ch-ua-mobile".to_owned(), "?0".to_owned()), + ("sec-ch-ua-platform".to_owned(), "\"Windows\"".to_owned()), + ], + } + ); + } + + /// The hints are in name order whatever order the browser sent them in, + /// because the evidence is part of the cache key. + #[test] + fn the_hints_are_in_name_order_whatever_order_they_came_in() { + let one = request_with(&[("sec-ch-ua-mobile", "?0"), ("sec-ch-ua", "\"A\"")]); + let other = request_with(&[("sec-ch-ua", "\"A\""), ("sec-ch-ua-mobile", "?0")]); + + assert_eq!(evidence_for(&one), evidence_for(&other)); + assert_eq!(evidence_for(&one).client_hints[0].0, "sec-ch-ua"); + } +} diff --git a/crates/shared/51degrees/src/hints_engine.rs b/crates/shared/51degrees/src/hints_engine.rs new file mode 100644 index 0000000000..57fe007086 --- /dev/null +++ b/crates/shared/51degrees/src/hints_engine.rs @@ -0,0 +1,481 @@ +//! The server side client hints question, asked on the license key. +//! +//! The question names the three properties the service names its `Accept-CH` +//! in, and a license key answers with exactly those. A resource key would be +//! spent on a call the browser never sees, and answers with everything it +//! carries whatever the call asks for. + +use std::sync::Arc; + +use trusted_server_core::evidence::RequestInfo; +use trusted_server_core::platform::RuntimeServices; + +use crate::client::{CloudPipeline, accept_ch_from_answer}; +use crate::engine_cache::AnswerCache; + +/// The properties the service names its `Accept-CH` in, and nothing else, +/// because a license key answers with exactly what the call names. Each is +/// read out of the answer by [`accept_ch_from_answer`], which the test +/// `the_values_asked_for_are_the_three_the_answer_is_read_from` holds. +pub const ACCEPT_CH_VALUES: &[&str] = &[ + "device.SetHeaderBrowserAccept-CH", + "device.SetHeaderHardwareAccept-CH", + "device.SetHeaderPlatformAccept-CH", +]; + +/// The base the data endpoint is under, being the configured endpoint without +/// its last path segment. On a multi-tenant service that segment carries the +/// resource key, so a license key call built from this base cannot present +/// it. +#[must_use] +pub fn engine_base(endpoint: &str) -> Option { + let mut url = url::Url::parse(endpoint).ok()?; + { + let mut segments = url.path_segments_mut().ok()?; + segments.pop(); + segments.push(""); + } + url.set_query(None); + url.set_fragment(None); + Some(url.into()) +} + +/// The server side client hints question, a reader of the one answer the +/// deployment's pipeline holds for a request's evidence. +#[derive(Debug, Clone)] +pub struct ClientHintsCall { + pipeline: Arc, +} + +impl ClientHintsCall { + /// The question on the deployment's pipeline, which [`ACCEPT_CH_VALUES`] + /// has been registered with. + #[must_use] + pub fn on(pipeline: Arc) -> Self { + Self { pipeline } + } + + /// The question on a pipeline of its own for `endpoint` and + /// `license_key`, holding its answers in `cache`, for a test or a + /// deployment that asks nothing else. + /// + /// Returns `None` when the endpoint is not a URL, which startup validation + /// refuses anyway. + #[must_use] + pub fn new( + endpoint: &str, + license_key: &str, + timeout_ms: u32, + cache: AnswerCache, + ) -> Option { + CloudPipeline::new(endpoint, license_key, timeout_ms, cache) + .map(|pipeline| Self::on(Arc::new(pipeline.wanting(ACCEPT_CH_VALUES)))) + } + + /// The `Accept-CH` value the service asks for from the request's browser, + /// or `None` when the request names no browser, the service names no + /// hints or the call could not be made. A failure is a logged line, never + /// an error the request carries. + /// + /// The evidence is the request's `User-Agent` and `Sec-CH-UA` hints alone, + /// so where a loading cache module holds the answer, the answer for one + /// reader serves every reader whose browser sends the same values. + pub async fn accept_ch( + &self, + request_info: &dyn RequestInfo, + services: &RuntimeServices, + ) -> Option { + let evidence = crate::evidence::evidence_for(request_info); + if evidence.user_agent.trim().is_empty() { + // The service names hints by the browser, and there is none to + // name them for. + return None; + } + match self.pipeline.answer(&evidence, services).await { + Ok(answer) => accept_ch_from_answer(&answer), + Err(error) => { + log::warn!("51Degrees client hints: no Accept-CH is sent. {error:?}"); + None + } + } + } +} + +/// The hint names the service asked the request's browser for, lower case, +/// as [`hint_names`] gives them. +/// +/// The client hints middleware leaves them in the request's state under +/// [`crate::BUILDER_ID`] as it filters a page request, and core copies them +/// into the document state the `[[serve]]` middleware share. A module that +/// wants to know what the browser was asked to send reads them there. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServiceHints(Vec); + +impl ServiceHints { + /// The names the service asked for. + #[must_use] + pub const fn new(names: Vec) -> Self { + Self(names) + } + + /// The names, in the service's order. + #[must_use] + pub fn names(&self) -> &[String] { + &self.0 + } +} + +/// The hint names an `Accept-CH` value lists, lower case, in its order and +/// each once. +/// +/// Lower case because the `Delegate-CH` content and a request's header names +/// are written that way, whatever case the service answers in. +#[must_use] +pub fn hint_names(accept_ch: &str) -> Vec { + let mut names: Vec = Vec::new(); + for name in accept_ch.split(',') { + let name = name.trim().to_ascii_lowercase(); + if !name.is_empty() && !names.contains(&name) { + names.push(name); + } + } + names +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_services::{MemoryKvStore, ScriptedHttpClient, services_with}; + use trusted_server_core::evidence::OwnedRequestInfo; + + /// What the data endpoint answers for a license key naming the three. + const DATA: &str = r#"{"device":{ + "setheaderbrowseraccept-ch":"Sec-CH-UA,Sec-CH-UA-Platform", + "setheaderhardwareaccept-ch":"Sec-CH-UA-Model", + "setheaderplatformaccept-ch":"Sec-CH-UA-Platform-Version"}}"#; + + fn chrome() -> OwnedRequestInfo { + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::USER_AGENT, + http::HeaderValue::from_static("Mozilla/5.0"), + ); + OwnedRequestInfo::new("2.125.160.216".to_owned(), headers) + } + + fn call() -> ClientHintsCall { + ClientHintsCall::new( + "https://cloud.example.com/api/v4/a-public-resource-key.json", + "a-secret-license-key", + 500, + crate::test_services::answer_cache(), + ) + .expect("should build") + } + + /// A server side call presents the license key and never the public + /// resource key. + #[tokio::test] + async fn the_call_presents_the_license_key_and_never_the_resource_key() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + + let accept_ch = call().accept_ch(&chrome(), &services).await; + + let sent = client.sent(); + let data = sent.last().expect("the data request should have been sent"); + assert!( + data.body.contains("license=a-secret-license-key"), + "got {}", + data.body + ); + assert!( + !data.body.contains("resource="), + "a resource key is public by design and a call the browser never sees has no \ + business spending it: {}", + data.body + ); + assert!( + !sent + .iter() + .any(|request| request.uri.contains("a-public-resource-key")), + "and the key in the configured endpoint path must not reach the address either: \ + {sent:?}" + ); + assert_eq!( + accept_ch.as_deref(), + Some("Sec-CH-UA, Sec-CH-UA-Platform, Sec-CH-UA-Model, Sec-CH-UA-Platform-Version"), + ); + } + + /// A license key names no properties of its own, so the call names them: + /// the three the hints are read from, among everything else the + /// deployment's modules registered, because one call carries them all. + #[tokio::test] + async fn the_call_names_the_three_properties_among_the_deployments() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + + call().accept_ch(&chrome(), &services).await; + + let sent = client.sent(); + let body = &sent.last().expect("the data request").body; + // One field carrying a comma separated list, as the service's `values` + // parameter expects. + let asked: Vec = body + .split('&') + .filter_map(|field| field.strip_prefix("values=")) + .flat_map(|list| list.split("%2C")) + .map(str::to_owned) + .collect(); + + for value in ACCEPT_CH_VALUES { + assert!( + asked.iter().any(|name| name == value), + "{value} in {asked:?}" + ); + } + assert_eq!(sent.len(), 1, "one data call"); + } + + /// A reader from `address`, on Chrome, carrying the `Sec-CH-UA` hints + /// Chrome sends unasked, cookies, and a header that is not a hint. + fn reader(address: &str, cookies: &'static str) -> OwnedRequestInfo { + let mut headers = http::HeaderMap::new(); + for (name, value) in [ + ("user-agent", "Mozilla/5.0"), + ("sec-ch-ua", "\"Chromium\";v=\"140\""), + ("sec-ch-ua-mobile", "?0"), + ("sec-ch-ua-platform", "\"Windows\""), + ("cookie", cookies), + ("accept-language", "en-GB"), + ] { + headers.insert( + http::HeaderName::from_static(name), + http::HeaderValue::from_static(value), + ); + } + OwnedRequestInfo::new(address.to_owned(), headers) + } + + /// The request the client builds carries the `User-Agent` and the + /// `Sec-CH-UA` hints, and neither the reader's address nor a cookie nor + /// any other header. + #[tokio::test] + async fn no_address_and_no_cookie_reaches_the_request_the_client_builds() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + + call() + .accept_ch( + &reader("2.125.160.216", "51D_ScreenPixelsWidth=1080; ts-ec=an-id"), + &services, + ) + .await; + + let sent = client.sent(); + assert_eq!(sent.len(), 1); + let body = &sent[0].body; + for field in [ + "user-agent=", + "sec-ch-ua=", + "sec-ch-ua-mobile=", + "sec-ch-ua-platform=", + ] { + assert!(body.contains(field), "{field} in {body}"); + } + let request = format!("{} {} {:?}", sent[0].uri, body, sent[0].headers); + for absent in [ + "client-ip", + "2.125.160.216", + "cookie", + "51D_", + "1080", + "ts-ec", + "an-id", + "accept-language", + "en-GB", + ] { + assert!( + !request + .to_ascii_lowercase() + .contains(&absent.to_ascii_lowercase()), + "{absent} reached the service: {request}" + ); + } + } + + /// Two readers on the same browser share one answer and one call, + /// whatever their addresses and cookies. + #[tokio::test] + async fn one_answer_serves_every_reader_on_the_same_browser() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + + let first = call() + .accept_ch(&reader("2.125.160.216", "ts-ec=one"), &services) + .await; + let second = call() + .accept_ch(&reader("192.0.2.1", "ts-ec=two"), &services) + .await; + + assert_eq!( + client.sent().len(), + 1, + "the second reader reads the held answer" + ); + assert!(first.is_some()); + assert_eq!(first, second); + } + + /// A request with no `User-Agent` names no browser for the service to + /// name hints for, so nothing is asked and nothing is sent. + #[tokio::test] + async fn a_request_with_no_user_agent_asks_nothing() { + let client = Arc::new(ScriptedHttpClient::new(&[DATA])); + let services = services_with(client.clone(), Arc::new(MemoryKvStore::default())); + let request = OwnedRequestInfo::new("2.125.160.216".to_owned(), http::HeaderMap::new()); + + assert!(call().accept_ch(&request, &services).await.is_none()); + assert!(client.sent().is_empty()); + } + + /// The answer is held, so a second instance sharing only the store asks + /// the cloud nothing. + #[tokio::test] + async fn a_second_instance_asks_the_cloud_nothing_at_all() { + let store = Arc::new(MemoryKvStore::default()); + let first = Arc::new(ScriptedHttpClient::new(&[DATA])); + call() + .accept_ch(&chrome(), &services_with(first.clone(), store.clone())) + .await; + assert_eq!(first.sent().len(), 1, "the first call pays for the answer"); + + // A separate instance, sharing only the store. + let second = Arc::new(ScriptedHttpClient::new(&[DATA])); + let accept_ch = call() + .accept_ch(&chrome(), &services_with(second.clone(), store)) + .await; + + assert_eq!( + second.sent().len(), + 0, + "the answer was paid for by the first instance, so a second one with the same \ + evidence has nothing to ask" + ); + assert!(accept_ch.is_some(), "and the answer still comes back"); + } + + /// A service that cannot be reached is no header, never a failed request. + #[tokio::test] + async fn an_unreachable_service_yields_no_header_and_no_error() { + let client = Arc::new(ScriptedHttpClient::new(&["not json at all"])); + let services = services_with(client, Arc::new(MemoryKvStore::default())); + + assert!( + call().accept_ch(&chrome(), &services).await.is_none(), + "a visitor whose browser was not asked for its hints is still served a page" + ); + } + + /// The base is the endpoint without its last path segment, which a + /// multi-tenant endpoint puts the public key in, and without a query or + /// fragment, because the data file is added to it and anything left on + /// the configured endpoint would be sent twice. + #[test] + fn the_base_is_the_endpoint_without_its_last_segment() { + for (case, endpoint, base) in [ + ( + "a resource key in the last segment", + "https://cloud.example.com/api/v4/AQS5HKcy.json", + Some("https://cloud.example.com/api/v4/"), + ), + ( + "a self-hosted container's endpoint, whose last segment is `json`", + "http://127.0.0.1:8080/api/v4/json", + Some("http://127.0.0.1:8080/api/v4/"), + ), + ( + "a query string", + "https://cloud.example.com/api/v4/key.json?resource=public", + Some("https://cloud.example.com/api/v4/"), + ), + ( + "a fragment", + "https://cloud.example.com/api/v4/key.json#top", + Some("https://cloud.example.com/api/v4/"), + ), + ("not a URL", "not-a-url", None), + ] { + assert_eq!(engine_base(endpoint).as_deref(), base, "{case}"); + } + } + + #[test] + fn an_endpoint_that_is_not_a_url_yields_no_call() { + assert!( + ClientHintsCall::new( + "not-a-url", + "license", + 500, + crate::test_services::answer_cache() + ) + .is_none() + ); + } + + /// Every value the call names is read out of the answer, each with a hint + /// of its own, so a value added here that the reader does not read fails. + /// The reader lowercases, because the service answers in lower case + /// element and property names. + #[test] + fn the_values_asked_for_are_the_three_the_answer_is_read_from() { + let mut device = serde_json::Map::new(); + let mut hints = Vec::new(); + for (index, value) in ACCEPT_CH_VALUES.iter().enumerate() { + let (element, property) = value + .split_once('.') + .expect("every value names its component"); + assert_eq!(element, "device", "{value} is read out of `device`"); + let hint = format!("Sec-CH-Test-{index}"); + device.insert(property.to_lowercase(), hint.clone().into()); + hints.push(hint); + } + let answer = crate::client::CloudAnswer::new(serde_json::json!({ "device": device })); + + assert_eq!( + accept_ch_from_answer(&answer), + Some(hints.join(", ")), + "what is asked for has to be what is read, or the call buys nothing" + ); + } + + /// The `Accept-CH` the live service answers a publisher's license key + /// with. + const SERVICE_ACCEPT_CH: &str = "Sec-CH-UA, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Mobile, \ + Sec-CH-UA-Platform, Sec-CH-UA-Model, Sec-CH-UA-Platform-Version"; + + #[test] + fn the_names_are_the_service_list_lower_case_in_its_order() { + assert_eq!( + hint_names(SERVICE_ACCEPT_CH), + vec![ + "sec-ch-ua".to_owned(), + "sec-ch-ua-full-version-list".to_owned(), + "sec-ch-ua-mobile".to_owned(), + "sec-ch-ua-platform".to_owned(), + "sec-ch-ua-model".to_owned(), + "sec-ch-ua-platform-version".to_owned(), + ] + ); + } + + #[test] + fn a_repeated_or_empty_name_is_written_once_or_not_at_all() { + assert_eq!( + hint_names(" Sec-CH-UA-Model ,, sec-ch-ua-model, Sec-CH-UA-Arch,"), + vec!["sec-ch-ua-model".to_owned(), "sec-ch-ua-arch".to_owned()] + ); + assert!(hint_names("").is_empty()); + } +} diff --git a/crates/shared/51degrees/src/lib.rs b/crates/shared/51degrees/src/lib.rs new file mode 100644 index 0000000000..a921538f75 --- /dev/null +++ b/crates/shared/51degrees/src/lib.rs @@ -0,0 +1,30 @@ +//! What the 51Degrees modules share. +//! +//! Each 51Degrees module lives in its own crate under its type. What more +//! than one of them needs lives here, being their shared table, the one call +//! to the cloud service and its held answer, the evidence the call sends and +//! the cache the answer is held in, so none depends on another and none of +//! it is in core. + +/// The id the 51Degrees builder and its registration carry. No configuration +/// names it, because `[shared.51degrees]` runs the builder and each module is +/// selected by its crate folder. +pub const BUILDER_ID: &str = "51degrees"; + +/// Folded into the dynamic backend name, so these crates' calls are not +/// confused with another caller's to the same host. +pub const BACKEND_DISCRIMINATOR: &str = BUILDER_ID; + +/// The value the service uses for a property it could not determine. +pub const UNKNOWN: &str = "Unknown"; + +pub mod client; +pub mod config; +pub mod engine_cache; +pub mod evidence; +pub mod hints_engine; +pub mod license; +/// Runtime services for the 51Degrees crates' tests, behind a feature so +/// nothing outside a test can reach them. +#[cfg(any(test, feature = "test-utils"))] +pub mod test_services; diff --git a/crates/shared/51degrees/src/license.rs b/crates/shared/51degrees/src/license.rs new file mode 100644 index 0000000000..cec849208e --- /dev/null +++ b/crates/shared/51degrees/src/license.rs @@ -0,0 +1,117 @@ +//! The license key every 51Degrees module presents. +//! +//! Every module takes its key from [`license_key`], through the builder that +//! registers them, so a missing key has one answer, which is that the +//! configuration is refused. + +use error_stack::Report; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::redacted::Redacted; +use trusted_server_core::settings::Settings; + +use crate::config::{self, vendor}; + +/// The license key every 51Degrees server side call presents, from +/// `[shared.51degrees]`. +/// +/// Fatal when absent, because a server side call presents the license key and +/// never the public resource key, and without one the deployment would serve +/// every page while answering nothing. A configuration without +/// `[shared.51degrees]` runs no 51Degrees module, so it never calls this. +/// +/// A configuration error is served from every route on a host that cannot +/// start without its settings, so the key has to be in the secret store before +/// a deploy that carries the table. +/// +/// # Errors +/// +/// A configuration error when there is no `[shared.51degrees]`, when it +/// cannot be read, or when it has no `license_key` or an empty one. +pub fn license_key(settings: &Settings) -> Result, Report> { + let vendor = vendor(); + let Some(shared) = config::read(settings)? else { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "There is no [shared.{vendor}], so there is no 51Degrees license key for a \ + 51Degrees module to present. Add [shared.{vendor}] with its `endpoint` and \ + `license_key`." + ), + })); + }; + shared + .license_key + .filter(|key| !key.expose().trim().is_empty()) + .ok_or_else(|| { + Report::new(TrustedServerError::Configuration { + message: format!( + "[shared.{vendor}] has no `license_key`. Every 51Degrees module presents a \ + license key on its server side calls, and never the public resource key in \ + `endpoint`, so the table needs one. Set `license_key`, or remove \ + [shared.{vendor}] and every 51Degrees module the configuration selects." + ), + }) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use trusted_server_core::html_processor::test_support::page_settings; + + #[test] + fn the_configured_key_is_the_one_every_module_presents() { + let settings = page_settings( + r#" + [shared.51degrees] + endpoint = "https://cloud.example.com/api/v4/json" + license_key = "a-license-key" + "#, + ); + + let key = license_key(&settings).expect("a configured key should be read"); + + assert_eq!(key.expose(), "a-license-key"); + } + + #[test] + fn a_shared_table_with_no_key_stops_the_deployment_naming_the_setting() { + for table in [ + r#"endpoint = "https://cloud.example.com/api/v4/json""#, + r#"endpoint = "https://cloud.example.com/api/v4/json" + license_key = " ""#, + ] { + let settings = page_settings(&format!( + r#" + [shared.51degrees] + {table} + "# + )); + + let message = format!( + "{:?}", + license_key(&settings).expect_err("there is no key to present") + ); + + assert!( + message.contains("[shared.51degrees] has no `license_key`"), + "the message has to name the setting to fix: {message}" + ); + } + } + + #[test] + fn a_configuration_without_the_shared_table_has_no_key_to_give() { + let settings = page_settings(""); + + let message = format!( + "{:?}", + license_key(&settings).expect_err("51Degrees is not configured") + ); + + assert!( + message.contains("There is no [shared.51degrees]"), + "the message has to say why there is no key rather than blame the setting: \ + {message}" + ); + } +} diff --git a/crates/shared/51degrees/src/test_services.rs b/crates/shared/51degrees/src/test_services.rs new file mode 100644 index 0000000000..d278f8799a --- /dev/null +++ b/crates/shared/51degrees/src/test_services.rs @@ -0,0 +1,470 @@ +//! Runtime services and stand-ins for the tests of every 51Degrees crate. +//! +//! [`RuntimeServices`] needs several things set before it will build, and +//! every 51Degrees module takes one. Written once here rather than in each +//! crate's tests, so a test that wants to prove a call was not made can do so +//! by counting rather than by inferring it from a failure that looks the same +//! as the outage case. + +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use edgezero_core::key_value_store::{KvError, KvPage, KvStore}; +use error_stack::Report; +use trusted_server_core::platform::{ + BackendNamingPolicy, ClientInfo, DisabledGeo, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformGeoModule, PlatformHttpClient, PlatformHttpRequest, + PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, + PlatformValueCache, RuntimeServices, StoreId, StoreName, UnavailableKvStore, +}; + +/// Stores that answer nothing. No 51Degrees module reads either. +pub struct StubStore; + +impl PlatformConfigStore for StubStore { + fn get(&self, _store_name: &StoreName, _key: &str) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +impl PlatformSecretStore for StubStore { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +/// A backend that resolves, so what a test proves is the call itself and not +/// the registration in front of it. +pub struct ResolvingBackend; + +impl PlatformBackend for ResolvingBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Axum + } + + fn predict_name(&self, _spec: &PlatformBackendSpec) -> Result> { + Ok("fiftyone_degrees_test".to_owned()) + } + + fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { + Ok("fiftyone_degrees_test".to_owned()) + } +} + +/// A key-value store held in memory for the life of one test. +/// +/// Entries never expire, so what a test reads back does not depend on the +/// clock. The time to live each entry was written with is kept, so a test can +/// read it with [`MemoryKvStore::lifetime`]. +#[derive(Debug, Default)] +pub struct MemoryKvStore { + entries: Mutex>, + lifetimes: Mutex>, +} + +impl MemoryKvStore { + /// The time to live `key` was last written with, or `None` when it was + /// written without one or not at all. + /// + /// # Panics + /// + /// When the lock is poisoned, which only a panicking test causes. + #[must_use] + pub fn lifetime(&self, key: &str) -> Option { + self.lifetimes + .lock() + .expect("should lock the test store") + .get(key) + .copied() + } + + /// Every key written, sorted, so a test can say what was written and what + /// was not. + /// + /// # Panics + /// + /// When the store's lock is poisoned, which only a panicking test causes. + #[must_use] + pub fn keys(&self) -> Vec { + let mut keys: Vec = self + .entries + .lock() + .expect("should lock the test store") + .keys() + .cloned() + .collect(); + keys.sort(); + keys + } +} + +#[async_trait::async_trait(?Send)] +impl KvStore for MemoryKvStore { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + Ok(self + .entries + .lock() + .expect("should lock the test store") + .get(key) + .cloned()) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.entries + .lock() + .expect("should lock the test store") + .insert(key.to_owned(), value); + self.lifetimes + .lock() + .expect("should lock the test store") + .remove(key); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + self.put_bytes(key, value).await?; + self.lifetimes + .lock() + .expect("should lock the test store") + .insert(key.to_owned(), ttl); + Ok(()) + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.entries + .lock() + .expect("should lock the test store") + .remove(key); + self.lifetimes + .lock() + .expect("should lock the test store") + .remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Err(KvError::Unavailable) + } +} + +/// A value cache held in memory, standing in for a host's shared cache. +/// +/// Entries never expire. The lifetime each was put with is kept, so a test can +/// read it with [`MemoryValueCache::lifetime`]. +#[derive(Debug, Default)] +pub struct MemoryValueCache { + entries: Mutex>, +} + +impl MemoryValueCache { + /// How many entries are held. + /// + /// # Panics + /// + /// When the lock is poisoned, which only a panicking test causes. + pub fn len(&self) -> usize { + self.entries + .lock() + .expect("should lock the test cache") + .len() + } + + /// Whether nothing is held. + /// + /// # Panics + /// + /// As [`MemoryValueCache::len`]. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The lifetime `key` was last put with, or `None` when it is not held. + /// + /// # Panics + /// + /// As [`MemoryValueCache::len`]. + #[must_use] + pub fn lifetime(&self, key: &str) -> Option { + self.entries + .lock() + .expect("should lock the test cache") + .get(key) + .map(|(_, lifetime)| *lifetime) + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformValueCache for MemoryValueCache { + async fn get(&self, key: &str) -> Option { + self.entries + .lock() + .expect("should lock the test cache") + .get(key) + .map(|(value, _)| value.clone()) + } + + async fn put(&self, key: &str, value: Bytes, lifetime: Duration) { + self.entries + .lock() + .expect("should lock the test cache") + .insert(key.to_owned(), (value, lifetime)); + } +} + +/// One request as it left this deployment. +#[derive(Debug, Clone)] +pub struct SentRequest { + /// The method sent. + pub method: String, + /// The address asked for, which is where a credential in a query string + /// would show up. + pub uri: String, + /// The body sent, which is where the form fields of a data request are. + pub body: String, + /// The headers sent, by name and value, which is where a credential sent + /// in a header would show up. + pub headers: Vec<(String, String)>, +} + +/// One answer a [`ScriptedHttpClient`] gives, being a status and a body. +#[derive(Debug, Clone)] +pub struct ScriptedAnswer { + status: u16, + body: String, +} + +impl ScriptedAnswer { + /// `body` answered with `status`. + #[must_use] + pub fn new(status: u16, body: &str) -> Self { + Self { + status, + body: body.to_owned(), + } + } +} + +/// An HTTP client that answers a queue of answers in order and keeps what it +/// was asked. Once the queue runs out it answers `200` with `{}`. +#[derive(Debug)] +pub struct ScriptedHttpClient { + answers: Mutex>, + sent: Mutex>, + delay: Option, +} + +impl ScriptedHttpClient { + /// A client answering `bodies` in order, each with `200`. + #[must_use] + pub fn new(bodies: &[&str]) -> Self { + Self::answering(bodies.iter().map(|body| ScriptedAnswer::new(200, body))) + } + + /// A client giving `answers` in order, each with its own status. + #[must_use] + pub fn answering(answers: impl IntoIterator) -> Self { + Self { + answers: Mutex::new(answers.into_iter().collect()), + sent: Mutex::new(Vec::new()), + delay: None, + } + } + + /// Holds each answer back for `delay`, blocking the thread that sent the + /// request, as a slow service would, so a test can have other requests + /// arrive while one is waiting. + #[must_use] + pub fn with_delay(mut self, delay: Duration) -> Self { + self.delay = Some(delay); + self + } + + /// Everything this client was asked for, in order. + /// + /// # Panics + /// + /// When the log's lock is poisoned, which only a panicking test causes. + pub fn sent(&self) -> Vec { + self.sent.lock().expect("should lock the sent log").clone() + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformHttpClient for ScriptedHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + let method = request.request.method().to_string(); + let uri = request.request.uri().to_string(); + let headers = request + .request + .headers() + .iter() + .map(|(name, value)| { + ( + name.as_str().to_owned(), + value.to_str().unwrap_or_default().to_owned(), + ) + }) + .collect(); + let body = request + .request + .into_body() + .into_bytes_bounded(64 * 1024) + .await + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .unwrap_or_default(); + self.sent + .lock() + .expect("should lock the sent log") + .push(SentRequest { + method, + uri, + body, + headers, + }); + if let Some(delay) = self.delay { + std::thread::sleep(delay); + } + let next = self + .answers + .lock() + .expect("should lock the script") + .pop_front() + .unwrap_or_else(|| ScriptedAnswer::new(200, "{}")); + let response = http::Response::builder() + .status(next.status) + .header(http::header::CONTENT_TYPE, "application/json") + .body(EdgeBody::from(next.body.into_bytes())) + .expect("should build the test response"); + Ok(PlatformResponse::new(response)) + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn select( + &self, + _pending: Vec, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +/// The loading cache a test's services hold answers in: the 51Degrees +/// module's, which a deployment links, so what one call holds is what the +/// next one reads, with a memory copy of its own for each set of services, as +/// each instance has. +fn loading_caches() -> Arc { + Arc::new(trusted_server_core::platform::LoadingCaches::every( + Arc::new(trusted_server_loading_cache_51degrees::LoadingCaches::new()), + )) +} + +/// Services whose cloud calls are answered by `client`, sharing `store`. +pub fn services_with( + client: Arc, + store: Arc, +) -> RuntimeServices { + RuntimeServices::builder() + .permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()) + .config_store(Arc::new(StubStore)) + .secret_store(Arc::new(StubStore)) + .kv_store(store) + .backend(Arc::new(ResolvingBackend)) + .http_client(client) + .geo(Arc::new(PlatformGeoModule::new( + "none", + Arc::new(DisabledGeo), + ))) + .client_info(ClientInfo::default()) + .loading_caches(loading_caches()) + .prefixes( + trusted_server_core::prefix::test_support::trusted_server_for("publisher.example"), + ) + .build() +} + +/// Services whose cloud calls are answered by `client`, with `cache` as the +/// host's value cache and no key-value store at all, as on a host whose store +/// is not configured. +pub fn services_with_value_cache( + client: Arc, + cache: Arc, +) -> RuntimeServices { + RuntimeServices::builder() + .permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()) + .config_store(Arc::new(StubStore)) + .secret_store(Arc::new(StubStore)) + .kv_store(Arc::new(UnavailableKvStore)) + .value_cache(cache) + .backend(Arc::new(ResolvingBackend)) + .http_client(client) + .geo(Arc::new(PlatformGeoModule::new( + "none", + Arc::new(DisabledGeo), + ))) + .client_info(ClientInfo::default()) + .loading_caches(loading_caches()) + .prefixes( + trusted_server_core::prefix::test_support::trusted_server_for("publisher.example"), + ) + .build() +} + +/// The answer cache a test's cloud calls hold their answers in, with the +/// default lifetimes. +#[must_use] +pub fn answer_cache() -> crate::engine_cache::AnswerCache { + crate::engine_cache::AnswerCache::new( + &trusted_server_core::settings::LoadingCacheConfig::default(), + ) +} diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index ab081dd128..5d65e36095 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -31,6 +31,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "ti trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } +trusted-server-middleware-51degrees = { workspace = true } trusted-server-auction-aps = { workspace = true } trusted-server-auction-prebid = { workspace = true } trusted-server-auction-prebid-server = { workspace = true } diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 9fca1852c6..d7af2dc56c 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -136,6 +136,8 @@ pub fn shipped_module_builders() -> Vec { trusted_server_ad_tag_google::diagnostics::builder(), trusted_server_ads_txt_iabtl::builder(), trusted_server_headers_geo::builder(), + // Run by `[shared.51degrees]` rather than selected by a section. + trusted_server_middleware_51degrees::builder(), // Implementations `[demand]` and `[ad-server]` can name, which no // section selects. trusted_server_auction_protocol_openrtb::builder(), @@ -1162,6 +1164,7 @@ mod tests { "ad-tag.google.diagnostics", "ads-txt.iabtl", "headers.geo", + "51degrees", "auction-protocol.openrtb", "auction.prebid-server", "auction.aps", diff --git a/crates/trusted-server-adapter-axum/tests/seam_probe.rs b/crates/trusted-server-adapter-axum/tests/seam_probe.rs index bf8ea14fc0..2a8b23bb5f 100644 --- a/crates/trusted-server-adapter-axum/tests/seam_probe.rs +++ b/crates/trusted-server-adapter-axum/tests/seam_probe.rs @@ -15,7 +15,7 @@ use axum::http::{HeaderMap, Request}; use edgezero_adapter_axum::service::EdgeZeroAxumService; use error_stack::Report; use tower::{Service as _, ServiceExt as _}; -use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_adapter_axum::app::{TrustedServerApp, module_builders_with}; use trusted_server_core::auction::compile_auction_plan; use trusted_server_core::config::validate_settings_for_deploy_with; use trusted_server_core::error::TrustedServerError; @@ -644,7 +644,9 @@ async fn a_cache_table_no_module_declares_is_refused_by_the_deploy_check_and_at_ )); let both = [seam_probe::builder(), seam_probe::loading_cache_builder()]; - let deploy = validate_settings_for_deploy_with(&settings, &both) + // The deploy check reads the builders startup links, the adapter's own + // and the deployment's after them. + let deploy = validate_settings_for_deploy_with(&settings, &module_builders_with(&both)) .expect_err("the deploy check should refuse the misspelt table"); let startup = TrustedServerApp::routes_with_registrations(settings, &both) .err() @@ -658,7 +660,7 @@ async fn a_cache_table_no_module_declares_is_refused_by_the_deploy_check_and_at_ "should name the table: {message}" ); assert!( - message.contains("robots-txt, iabtl-answers"), + message.contains("robots-txt, 51degrees-answers, iabtl-answers"), "should list the caches this build declares: {message}" ); assert_eq!( diff --git a/crates/trusted-server-adapter-axum/tests/shipped_modules.rs b/crates/trusted-server-adapter-axum/tests/shipped_modules.rs index f0b06fcc30..7b345ed36e 100644 --- a/crates/trusted-server-adapter-axum/tests/shipped_modules.rs +++ b/crates/trusted-server-adapter-axum/tests/shipped_modules.rs @@ -139,12 +139,15 @@ fn documented_module_tables_validate_when_uncommented_and_selected() { /// Every linked page module refuses a setting it does not know, so a /// misspelt key in its table fails deploy validation naming the table and -/// the key, where it would otherwise be ignored. +/// the key, where it would otherwise be ignored. The planted table names a +/// key for each secret setting the module declares, so the unknown setting +/// is its only fault. #[test] fn every_linked_module_rejects_a_setting_it_does_not_know() { use serde_json::json; use trusted_server_core::config::validate_settings_for_deploy_with; use trusted_server_core::module_name::short_form; + use trusted_server_core::settings::SHARED_TYPE; use trusted_server_core::test_support::tests::create_test_settings; let linked = shipped_module_builders(); @@ -152,24 +155,41 @@ fn every_linked_module_rejects_a_setting_it_does_not_know() { .iter() .filter(|builder| builder.supplies_integration()) { - let name = builder - .module_name() - .expect("every linked page module names itself"); - let section = builder - .section() - .expect("every linked page module has a section"); + let mut planted = json!({ "no_such_setting": true }); + for secret in builder.secret_settings() { + if let Some((last, parents)) = secret.path.split_last() { + let mut at = &mut planted; + for parent in parents { + at = &mut at[*parent]; + } + at[*last] = json!("example-secret-name"); + } + } + let mut settings = create_test_settings(); - settings - .insert_module_config(section, name, &json!({ "no_such_setting": true })) - .expect("should insert the planted table"); + let (name, table) = if let Some(vendor) = builder.shared_settings() { + settings.shared = serde_json::from_value(json!({ vendor: planted })) + .expect("should plant the shared table"); + (builder.id(), format!("[{SHARED_TYPE}.{vendor}]")) + } else { + let name = builder + .module_name() + .expect("every linked page module names itself"); + let section = builder + .section() + .expect("every linked page module has a section"); + settings + .insert_module_config(section, name, &planted) + .expect("should insert the planted table"); + (name, format!("[{section}.{}]", short_form(section, name))) + }; let error = match validate_settings_for_deploy_with(&settings, &linked) { Ok(()) => panic!("`{name}` should refuse a setting it does not know"), Err(error) => format!("{error:?}"), }; - let written = short_form(section, name); assert!( - error.contains(&format!("[{section}.{written}]")) && error.contains("no_such_setting"), + error.contains(&table) && error.contains("no_such_setting"), "`{name}` should name its table and the unknown setting: {error}" ); } diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index 324ef21d31..bbbda065d1 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -32,6 +32,7 @@ serde_json = { workspace = true } trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } +trusted-server-middleware-51degrees = { workspace = true } trusted-server-auction-aps = { workspace = true } trusted-server-auction-prebid = { workspace = true } trusted-server-auction-prebid-server = { workspace = true } diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 688efa51b9..7ee42969ff 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -341,6 +341,8 @@ pub fn shipped_module_builders() -> Vec { trusted_server_ad_tag_google::diagnostics::builder(), trusted_server_ads_txt_iabtl::builder(), trusted_server_headers_geo::builder(), + // Run by `[shared.51degrees]` rather than selected by a section. + trusted_server_middleware_51degrees::builder(), // Implementations `[demand]` and `[ad-server]` can name, which no // section selects. trusted_server_auction_protocol_openrtb::builder(), @@ -1368,6 +1370,7 @@ mod tests { "ad-tag.google.diagnostics", "ads-txt.iabtl", "headers.geo", + "51degrees", "auction-protocol.openrtb", "auction.prebid-server", "auction.aps", diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 512d816322..5c869e96ec 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -36,6 +36,7 @@ serde_json = { workspace = true } trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } +trusted-server-middleware-51degrees = { workspace = true } trusted-server-auction-aps = { workspace = true } trusted-server-auction-prebid = { workspace = true } trusted-server-auction-prebid-server = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index f96641a9a5..711fcf8569 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -449,6 +449,8 @@ pub fn shipped_module_builders() -> Vec { trusted_server_ad_tag_google::diagnostics::builder(), trusted_server_ads_txt_iabtl::builder(), trusted_server_headers_geo::builder(), + // Run by `[shared.51degrees]` rather than selected by a section. + trusted_server_middleware_51degrees::builder(), // Implementations `[demand]` and `[ad-server]` can name, which no // section selects. trusted_server_auction_protocol_openrtb::builder(), @@ -3530,6 +3532,7 @@ mod tests { "ad-tag.google.diagnostics", "ads-txt.iabtl", "headers.geo", + "51degrees", "auction-protocol.openrtb", "auction.prebid-server", "auction.aps", diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index 8a10f977ae..b50dcbdc6b 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -34,6 +34,7 @@ log = { workspace = true } trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } +trusted-server-middleware-51degrees = { workspace = true } trusted-server-auction-aps = { workspace = true } trusted-server-auction-prebid = { workspace = true } trusted-server-auction-prebid-server = { workspace = true } diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 875d637d37..57148f8f92 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -353,6 +353,8 @@ pub fn shipped_module_builders() -> Vec { trusted_server_ad_tag_google::diagnostics::builder(), trusted_server_ads_txt_iabtl::builder(), trusted_server_headers_geo::builder(), + // Run by `[shared.51degrees]` rather than selected by a section. + trusted_server_middleware_51degrees::builder(), // Implementations `[demand]` and `[ad-server]` can name, which no // section selects. trusted_server_auction_protocol_openrtb::builder(), @@ -1973,6 +1975,7 @@ mod tests { "ad-tag.google.diagnostics", "ads-txt.iabtl", "headers.geo", + "51degrees", "auction-protocol.openrtb", "auction.prebid-server", "auction.aps", diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 87e034a724..6ade072a65 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -61,6 +61,7 @@ tracing = { workspace = true } trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } +trusted-server-middleware-51degrees = { workspace = true } trusted-server-auction-aps = { workspace = true } trusted-server-auction-prebid = { workspace = true } trusted-server-auction-prebid-server = { workspace = true } diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs index 4ee6c419ea..17a36d28d0 100644 --- a/crates/trusted-server-cli/src/app_config.rs +++ b/crates/trusted-server-cli/src/app_config.rs @@ -55,6 +55,8 @@ pub fn linked_builders() -> Vec { trusted_server_ad_tag_google::diagnostics::builder(), trusted_server_ads_txt_iabtl::builder(), trusted_server_headers_geo::builder(), + // Run by `[shared.51degrees]` rather than selected by a section. + trusted_server_middleware_51degrees::builder(), // Implementations `[demand]` and `[ad-server]` can name, which no // section selects. trusted_server_auction_protocol_openrtb::builder(), @@ -278,6 +280,7 @@ mod tests { "ad-tag.google.diagnostics", "ads-txt.iabtl", "headers.geo", + "51degrees", "auction-protocol.openrtb", "auction.prebid-server", "auction.aps", diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index eb974dd55c..4442942be2 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -2121,6 +2121,16 @@ the service starts, with the names that could be written. | `[cache.robots-txt]` | `store_seconds` | `604800`, a week | 1 to 31536000 | The longest a module's answer is used. The module's own refresh ends it sooner. | | `[cache.robots-txt]` | `store_idle_seconds` | `2592000`, 30 days | 1 to 31536000 | How long an answer in the shared store lasts unused. | +The 51Degrees modules declare `51degrees-answers`, the cache that holds the +service's answers, sized in `[cache.51degrees-answers]`. + +| Table | Field | Default | Range | Description | +| --------------------------- | -------------------- | ---------------- | ------------- | ----------------------------------------------------------------------- | +| `[cache.51degrees-answers]` | `module` | none | | The module whose loading cache holds the 51Degrees service's answers. | +| `[cache.51degrees-answers]` | `memory_seconds` | `60` | 0 to 31536000 | How long the copy in memory is used. Zero keeps none. | +| `[cache.51degrees-answers]` | `store_seconds` | `604800`, a week | 1 to 31536000 | The longest an answer is used. An answer ends after an hour regardless. | +| `[cache.51degrees-answers]` | `store_idle_seconds` | `86400`, a day | 1 to 31536000 | How long an answer in the shared store lasts unused. | + A module may declare caches of its own, named by the module, and a deployment selects a loading cache for each in its own `[cache.]` table with the same four settings. Core applies none of the lifetimes in such a table. They @@ -2234,6 +2244,7 @@ validation knows is listed below. | `[tag.google-tag-manager]` | [Google Tag Manager](/guide/integrations/google_tag_manager) | | `[ad-tag.google]` | [GPT](/guide/integrations/gpt) | | `[ad-tag.google.diagnostics]` | [GPT diagnostics](/guide/integrations/gpt-diagnostics) | +| `[shared.51degrees]` | [51Degrees](/guide/integrations/51degrees) | | `[cmp.inmobi]` | [InMobi Choice](/guide/integrations/inmobi) | | `[proxy.js_asset_proxy]` | [JS Asset Proxy](#js-asset-proxy-integration) (no dedicated guide) | | `[identity.lockr]` | [lockr](/guide/integrations/lockr) | @@ -2423,6 +2434,7 @@ generated configuration places it in. | `ad-tag.google.diagnostics` | `[[serve]]` | For a request that activated diagnostics, writes the bootstrap ahead of the script bundle and the tag that loads the diagnostics module straight after it | | `ads-txt.iabtl` | `[[fetch]]` | Writes the lines `[ads-txt.iabtl]` gives at the end of the publisher's `ads.txt`, on `text/plain` alone | | `headers.geo` | `[[serve]]` | Writes the location the request resolved as the `x-geo-*` response headers, or `x-geo-info-available: false` where none is known. `[headers] modules = ["geo"]` selects it | +| `51degrees.client_hints` | `[[serve]]` | Asks the 51Degrees service which client hints the reader's browser can send, writes `Accept-CH` and `Critical-CH` unless the browser already sends those hints, `Permissions-Policy` on the response and the `Delegate-CH` tag in the head for that list, and leaves the names in the request's state | | `robots_txt` | `[[fetch]]` | Makes the `robots.txt` that `[robots-txt]` describes, in place of whatever the origin answers, on `text/plain` and at `/robots.txt` alone | | `service_version` | `[[serve]]` | Writes `X-TS-Version`, the git version the build compiled in, and `X-TS-Service-Version`, the platform's version number of the service, when each is known. Core supplies it, so no section selects it | @@ -2516,6 +2528,35 @@ fetch middleware writes ahead of the script bundle, and so is naming it for one page in both phases, which would write it twice. See [InMobi Choice](/guide/integrations/inmobi) for the tag's parts and what the module keeps to. +### 51Degrees Integration + +**Section**: `[shared.51degrees]`, with `[middleware.51degrees.client_hints]` +and `[cache.51degrees-answers]` + +The table's presence runs the 51Degrees modules, and its fields are what +every one of them shares. + +| Field | Type | Default | Contract | +| ------------- | ------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `endpoint` | URL | Required | The service's JSON endpoint, with any resource key the deployment requires in its path, whose origin the browser's hints are delegated to | +| `license_key` | String | Required | The name of the license key's entry in the secret store, which every server side call presents in place of the public resource key | +| `timeout_ms` | Integer | `500` | How long a call to the service waits, `1..=10000` milliseconds | + +The client hints middleware's own table has one field. + +| Table | Field | Type | Default | Contract | +| ------------------------------------- | ---------- | ------- | ------- | -------------------------------------------------------------------------------------------- | +| `[middleware.51degrees.client_hints]` | `critical` | Boolean | `true` | Whether `Critical-CH` is written with `Accept-CH`, so a browser retries its first navigation | + +The service's answers load through the cache `51degrees-answers`, selected and +sized in `[cache.51degrees-answers]` with the four settings every declared +cache takes, see [Loading caches](#loading-caches), and are held only where +the build links a loading cache module. An answer ends after an hour whatever +`store_seconds` says. Each table refuses a field it does not +read, and a table with no `license_key` is refused, because every 51Degrees +module presents one. See [51Degrees](/guide/integrations/51degrees) for what +the middleware writes and where to place it. + ### Google Tag Integration **Section**: `[tag.google-tag]` diff --git a/docs/guide/integrations-overview.md b/docs/guide/integrations-overview.md index a8b8744363..ecf8c2711c 100644 --- a/docs/guide/integrations-overview.md +++ b/docs/guide/integrations-overview.md @@ -35,6 +35,7 @@ their operational procedures. | [`google_tag_manager`](/guide/integrations/google_tag_manager) | production | `[tag] modules = ["google-tag-manager"]` | settings builder | bundled | | [`gpt`](/guide/integrations/gpt) | production | `[ad-tag] modules = ["google"]` | settings builder | bundled | | [`gpt_diagnostics`](/guide/integrations/gpt-diagnostics) | development | `[ad-tag] modules = ["google.diagnostics"]` | settings builder | standalone | +| [`51degrees`](/guide/integrations/51degrees) | development | `[shared.51degrees]` | settings builder | none | | [`inmobi`](/guide/integrations/inmobi) | development | `[cmp] module = "inmobi"` | settings builder | none | | `js_asset_proxy` (no dedicated guide) | development | `[proxy] modules = ["js_asset_proxy"]` | settings builder | none | | [`lockr`](/guide/integrations/lockr) | production | `[identity] module = "lockr"` | settings builder | bundled | diff --git a/docs/guide/integrations/51degrees.md b/docs/guide/integrations/51degrees.md new file mode 100644 index 0000000000..8253f6a4d5 --- /dev/null +++ b/docs/guide/integrations/51degrees.md @@ -0,0 +1,149 @@ +# 51Degrees Integration + +**Category**: Device detection + +**Status**: Development + +**Type**: Client hints middleware, with the table and the service call the 51Degrees modules share + +## Overview + +A browser sends its high entropy client hints only when the site asks for +them, with `Accept-CH` on the response and `Critical-CH` when the first +navigation should be retried carrying them, and they reach a third party only +when the top level page delegates them, by a `Delegate-CH` tag or a +`Permissions-Policy` header. A `Delegate-CH` tag added by JavaScript is +ignored, so the tag has to be in the page as served. + +The `51degrees.client_hints` middleware writes all of that, on each page a +`[[serve]]` entry names it for, from the list the 51Degrees service gives for +the reader's browser. No hint name is written down in the code. As it filters +a page request, the middleware asks the service which hints it wants from +that browser, through the cache the 51Degrees modules share, and leaves the +names in the request's state. On the way out it writes: + +1. `Accept-CH` naming the hints, and `Critical-CH` naming the same unless + `critical` is off, so the browser retries the first navigation of a + session carrying them. +2. `Permissions-Policy` delegating each hint to the service's origin, as + `ch-ua-model=(self "https://cloud.51degrees.com")`, which keeps the hint + for the page's own origin as well. +3. The `Delegate-CH` tag in the head, in the shape 51Degrees publishes, being + the lower case hint names each followed by the origin and separated by + semicolons. + +Each header keeps every line the response already carries under that name, +with the hints added after them, so a policy the publisher's origin sets is +kept and a hint the origin already asks for is named once. A delegation comes +after the origin's own policy members, so for a hint the service names it +overrides an origin policy that denies the hint, such as `ch-ua-model=()`. A +browser engine the service names no hints for, such as Firefox or Safari, is +given none of them. A request that is not a page navigation asks the service +nothing, and a page navigation is a `GET` or `HEAD` with +`Sec-Fetch-Dest: document` or an `Accept` naming `text/html`. + +A browser keeps the hints an origin asked for until a response from that +origin names a different set, so `Accept-CH` and `Critical-CH` are left off +the response to a request that already carries every hint the service names. +They are still written where the response names an `Accept-CH` set of its +own, because that set would replace the one the browser holds. +`Permissions-Policy` and the `Delegate-CH` tag apply to their own page only, +so every page gets them. + +The origin the hints are delegated to is the origin of the configured +`endpoint`, so a deployment using a private cloud delegates to that, and an +endpoint on the page's own origin delegates nothing. + +The hint names stay in the request's state under the 51Degrees registration, +as `ServiceHints` from `trusted-server-shared-51degrees`, for a 51Degrees +module that wants to know what the browser was asked to send. + +## Configuration + +```toml +[shared.51degrees] +endpoint = "https://cloud.51degrees.com/api/v4/json" +license_key = "fiftyone_degrees_license_key" +# timeout_ms = 500 + +# [middleware.51degrees.client_hints] +# critical = true + +# [cache.51degrees-answers] +# Refused unless the build links crates/loading-cache/51degrees. +# module = "51degrees" + +[[serve]] +media_type = "text/html" +middleware = ["51degrees.client_hints"] +``` + +The table's presence runs the 51Degrees modules, so there is no `module` +line to write. Every one of them presents the license key on its server side +calls, and never the public resource key, so the table needs one. + +| Field | Type | Default | Contract | +| ------------- | ------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `endpoint` | URL | Required | The service's JSON endpoint, with any resource key the deployment requires in its path, whose origin the browser's hints are delegated to | +| `license_key` | String | Required | The name of the license key's entry in the secret store | +| `timeout_ms` | Integer | `500` | How long a call to the service waits, `1..=10000` milliseconds | + +The middleware's own table, `[middleware.51degrees.client_hints]`, has one +field, `critical`, on by default. Each table refuses a field it does not +read. + +The service's answers load through the cache `51degrees-answers`, and +`[cache.51degrees-answers]` names a module and sizes the copies, as for every +declared cache. A deployment that links one loading cache module holds the +answers there with no line in the document, each for an hour at most. A stock +build links none, so it asks the service on every page navigation the +middleware is placed for. + +## Placing the middleware + +The middleware writes its headers and tag where a `[[serve]]` entry names +`51degrees.client_hints`, and in no other phase, because the list is the +service's answer for one reader's browser and what the fetch phase leaves is +stored for every reader. The request filter asks the service only about a page +navigation whose path an entry naming the middleware covers, by the same rule +as the headers, so a deployment that names it in no entry sends no client +hints headers and no tag and asks the service nothing. Only the Fastly adapter +runs request filters, so on Axum, Cloudflare and Spin the middleware asks the +service nothing and writes nothing. The headers are written on every response +the publisher path sends for a path the entry covers, a redirect included, and +the tag on each page. + +A page the middleware leaves names for keeps to the origin path, because the +names are left in the request's state, which is what core does for every +request a module leaves a value on. That page is never read from a shared +template and never stored as one, and is sent `private, no-store`. + +The call sends the browser's `User-Agent` and its `Sec-CH-UA` hints, and +neither the reader's address nor a cookie. A call costs one round trip to the +service before the page. Where a loading cache module holds the answers, only +the first call for a `User-Agent` and set of those hints is made, and every +reader whose browser sends the same values then reads the held answer. On +Fastly the call fails after one second if it cannot connect, and +once connected it waits `timeout_ms` for the first byte and again for each gap +between bytes. A service that does not answer leaves the page with no headers +and no tag, and a logged line says why. + +## What is checked + +The table is refused, at load and by `ts config validate`, when it carries a +field it does not read, when `endpoint` is not a URL, when `timeout_ms` is +out of range, and when it has no `license_key` or an empty one. The +middleware's table is refused when it carries a field other than `critical`. +A `[cache.]` table naming a cache no module declares is refused with +the names that are declared. + +## Tests + +The module's own tests cover the headers on a document, `critical` off, a +document with no names, every line the response already carried, the tag's +shape and the origin it delegates to, the request filter against a scripted +service for a Chromium, a Gecko and a WebKit browser, an entry's `path` +deciding whether the service is asked, each table refusing a key it does not +know, and that no entry means no headers and no call. The shared crate's tests +cover the call's form, the credential it presents, that no address and no +cookie is sent, what it holds and what it refuses. diff --git a/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md b/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md new file mode 100644 index 0000000000..15232a0e9e --- /dev/null +++ b/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md @@ -0,0 +1,330 @@ +# Design Spec: A deployment using 51Degrees gets client hints from one middleware + +**Status:** Proposed, 2026-10-14. Raised as pull request IABTechLab/trusted-server#1386 (from jwrosewell/trusted-server#39), for Tech Lab review. It describes the stack from row 52 (IABTechLab/trusted-server#1410) up, where every adapter runs request filters. Below row 52 only the Fastly adapter runs them, as this row's changelog says. +**Author:** 51Degrees (contributed), for Tech Lab review +**Related specs:** `2026-10-12-shared-vendor-tables-design.md` (the `[shared.51degrees]` table), `2026-10-12-loading-cache-design.md` (the `51degrees-answers` cache), `2026-10-12-page-middleware-design.md` (the `[[serve]]` entries), `2026-10-14-source-modules-design.md` (the source a page is fetched from) and `2026-10-17-request-filters-on-every-adapter-design.md` (the request filters every adapter runs), all in IABTechLab/trusted-server#1342, which holds every design document of the stack +**Related PRs:** jwrosewell/trusted-server#39, IABTechLab/trusted-server#1386 +**Issues:** IABTechLab/trusted-server#1333 +**Last updated:** 2026-10-27 + +> **Why this spec exists.** A browser sends its high entropy client hints only when the site asks for them and delegates them, and nothing in the stack writes the headers and the tag that do so for a deployment that uses the 51Degrees service, so every first page view of a session is priced on the User-Agent alone. + +## 1. Problem + +A browser sends its high entropy client hints only when a response asks for +them with `Accept-CH`, retries the first navigation carrying them only when +`Critical-CH` names them, and lets a third party read them only when the top +level page delegates them, by a `Delegate-CH` tag in the page as served or a +`Permissions-Policy` header. The stack has the places to write each of those, +being the serve phase, the headers step and the head inserts, and no module +that does. + +The list of hints is the service's. The 51Degrees service names, for each +browser, the hints it wants in three properties, and the list changes as the +service does. A list written into a deployment drifts from it. + +In this document a deployment uses 51Degrees when its settings carry the +`[shared.51degrees]` table, because the stack has no 51Degrees device module +yet (section 3) and the table is what runs the modules that call the service. +In this document an origin on its own is a web origin, being a scheme, a host +and a port, and the server a publisher's pages come from is its source, as +`2026-10-14-source-modules-design.md` names it. + +## 2. Design + +**A serve middleware, `51degrees.client_hints`, in `crates/middleware/51degrees`.** +The crate registers the middleware and a request filter together. As the +filter sees a page navigation whose path a `[[serve]]` entry naming the +middleware covers, it asks the service, through the cache the 51Degrees +modules share, which hints it wants from the request's browser, and leaves the +names in the request's state. Core copies them into the document state the +serve middleware share. From that list the middleware writes +`Accept-CH`, `Critical-CH` and `Permissions-Policy` through +`Middleware::response_headers`, each adding to what the response already +carries under that name as "What the headers keep" says, and the +`Delegate-CH` tag in the head through its action. It reads `critical` from its +own table, `[middleware.51degrees.client_hints]`, on by default. +`Accept-CH` and `Critical-CH` are left off where the request already carries +every hint the service names and the response names no `Accept-CH` set of its +own, because a browser keeps an origin's set until a response names another. + +The three parts run in different places. + +| Part | Runs | Decided by | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------- | +| The filter, which asks the service and leaves names | Before routing, on every adapter, and it acts on page navigations only | Any `[[serve]]` entry naming the middleware whose `path` covers the request path, the rule the headers use | +| The `Delegate-CH` tag | In the serve phase, on each page | The first `[[serve]]` entry that covers the page's media type and path | +| `Accept-CH`, `Critical-CH` and `Permissions-Policy` | In the headers step of the serve phase, on each response the publisher path sends, a redirect included | Any `[[serve]]` entry whose `path` covers the request path, whatever its media type | + +A page navigation, for the filter, is a `GET` or `HEAD` request whose +`Sec-Fetch-Dest` is `document` or whose `Accept` names `text/html`, so a +request for a frame, or one a script makes for an HTML fragment, qualifies +when it sends such an `Accept`. Any other request asks the service nothing, +and so does a page navigation whose path no `[[serve]]` entry naming the +middleware covers, which is where no headers would be written. A service that +names no hints for a browser, or does not answer, leaves nothing, and the page +gets no headers and no tag. The origin delegated to is +the origin of the configured endpoint, and an endpoint on the page's own +origin delegates nothing. + +**What the headers keep.** For each of `Accept-CH`, `Critical-CH` and +`Permissions-Policy` the middleware reads every line the response carries +under that name, joins them in order into one list, and adds the hints after +it. `Accept-CH` and `Critical-CH` keep the response's hints and name each hint +once. `Permissions-Policy` keeps the response's members and adds one +delegation after them for each hint, as +`ch-ua-model=(self "https://cloud.51degrees.com")`. A later member of a name +replaces an earlier one in a policy, so for each hint the service names, the +delegation overrides a member of the publisher's own that denies the hint, +such as `ch-ua-model=()`. Core then writes each header with +`write_response_headers` (`crates/trusted-server-core/src/middleware.rs`), +which uses `HeaderMap::insert` and so replaces every line of that name with +the one line written, which already holds what each of them said. + +**What the 51Degrees modules share, in `crates/shared/51degrees`.** The +vendor's table, `[shared.51degrees]`, being `endpoint`, `license_key` and +`timeout_ms`, whose presence runs the 51Degrees builder. The license key +every server side call presents, and the refusal when the table has none. +The one call to the service for a request's evidence, a form POST to the +service's `json` endpoint over the platform's HTTP client, naming the +properties each module registered so one call carries them all, and its +answer loaded through the loading cache, so that where a deployment links a +loading cache module a question many requests ask at once is asked once, and +in a stock build, which links none, every call asks the service +(`2026-10-12-loading-cache-design.md`, section 7). The evidence a request +gives, being the `User-Agent` and the `Sec-CH-UA` hints and nothing else. The +names the service asked for, as the type a module reads them from the +request's state by. + +**The cache is a declared one.** The builder declares `51degrees-answers`, +and a deployment selects the loading cache that holds it and sizes its copies +in `[cache.51degrees-answers]`, as every declared cache is selected. A build +that links one loading cache module holds the answers without a line in the +document. A build that links no loading cache module, or several with no +`[cache.51degrees-answers]` table naming one, holds nothing, so every page +navigation an entry covers calls the service, and every stock build links +none. On Fastly the copies are held in the host's value cache, shared by every +instance at a location, with a brief +copy in process memory in front. An answer is held for an hour at most. The +key is a hash of the endpoint, the license key, the properties and the +evidence sent, being the `User-Agent` and the `Sec-CH-UA` hints, so one held +answer serves every reader whose browser sends the same values, and only the +first navigation from each such browser waits for the service. + +**The builder is run by the table, not selected by a section.** It carries +the vendor's shared table, declares the license key as a secret setting in +use whenever the table is present, and registers the filter and the +middleware together. The stock list ships it, so every adapter and the `ts` +command line tool take it, and every adapter runs its filter. + +```toml +[shared.51degrees] +endpoint = "https://cloud.51degrees.com/api/v4/json" +license_key = "fiftyone_degrees_license_key" + +[[serve]] +media_type = "text/html" +middleware = ["51degrees.client_hints"] +``` + +**The settings.** + +| Setting | Default | Contract | +| ---------------------------------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `[shared.51degrees] endpoint` | Required | A URL. The call goes to the endpoint with its last path segment replaced by `json`, so a resource key written in that segment is never sent, and a query or fragment on the endpoint is dropped. The hints are delegated to the endpoint's origin | +| `[shared.51degrees] license_key` | Required | The name of the license key's entry in the secret store. A table with none, or an empty one, is refused by the configuration check and when the application state is built | +| `[shared.51degrees] timeout_ms` | `500` | `1` to `10000` milliseconds. Only the Fastly adapter applies it, see "What bounds the call on each adapter" | +| `[middleware.51degrees.client_hints] critical` | `true` | `true` or `false` | +| `[cache.51degrees-answers]` | `memory_seconds` 60, `store_seconds` 604800 and `store_idle_seconds` 86400 | The four settings every declared cache takes. An answer ends after an hour whatever `store_seconds` says | + +`[shared.51degrees]` and `[middleware.51degrees.client_hints]` each refuse a +key they do not read. + +**What bounds the call on each adapter.** Every adapter runs request filters +(`2026-10-17-request-filters-on-every-adapter-design.md`), so the filter asks +the service on each of them. A module's time limit reaches an adapter only as +the timeouts of the backend it registers for the call, and only the Fastly +adapter applies them, so `timeout_ms` bounds the call there alone. + +| Adapter | What bounds the call | +| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Fastly | `timeout_ms` sets the first byte and between bytes timeouts of the backend, and the connection timeout is fixed at one second (`crates/trusted-server-adapter-fastly/src/backend.rs`) | +| Axum, a development adapter | The adapter's HTTP client ignores `timeout_ms` and allows 5 seconds to connect and 30 in all (`crates/trusted-server-adapter-axum/src/platform.rs`) | +| Cloudflare | The adapter's fetch sets no timeout, and the adapter's own note says only the Workers runtime's limit bounds a call (`crates/trusted-server-adapter-cloudflare/src/platform.rs`) | +| Spin | `spin_sdk::http::send` takes no timeout, so `timeout_ms` is ignored, and the adapter's own note says the Spin runtime's default applies (`crates/trusted-server-adapter-spin/src/platform.rs`) | + +**What each navigation sends.** For a page navigation an entry covers, the +filter asks the cache for the answer to one question, and where none is held +it makes one `POST` to the service. The form carries the license key, the +three properties that name the hints, and the evidence, being the +`User-Agent` and each `Sec-CH-UA` header the request carries, that is +`Sec-CH-UA` and every header whose name begins `Sec-CH-UA-`, under their own +names (`crates/shared/51degrees/src/evidence.rs`). Neither the visitor's +address nor a cookie nor any other header is sent. A request with no +`User-Agent` asks nothing. The filter reads only the three hint properties +from the answer. + +**Consent.** Every adapter hands every request filter the permissions +resolved for the request, so a filter can skip or narrow what it does when a +permission it depends on is unset (`docs/guide/permission-model.md`). The +51Degrees filter does not read them and the middleware declares no +permission, so the call is made whatever the permissions resolved for the +request say. The hints the browser then sends can also reach DataDome, where +its protection filter is enabled, because that filter builds its request to +DataDome's service from the `Sec-CH-UA`, `Sec-CH-UA-Arch`, +`Sec-CH-UA-Full-Version-List`, `Sec-CH-UA-Mobile`, `Sec-CH-UA-Model`, +`Sec-CH-UA-Platform` and `Sec-CH-Device-Memory` headers a request carries +(`crates/bot-protection/datadome/src/protection.rs`). + +**Open questions for the reviewer.** This change takes no position on two +things. + +The first is whether the call to the service needs a permission, and if it +does which one, with no call, no names and so no headers where it is not set. +A module names the data use it needs by an IAB Tech Lab Privacy Taxonomy Data +Use (`Permission` in `crates/trusted-server-core/src/permissions.rs`), and +none of them names device detection, client hints or a call that describes the +browser. The nearest are general ones, which the sample policy +(`config/permissions/sample.yaml`) treats in opposite ways. +`functional.performance` and `necessary.operations.website_use` are granted in +each of its three groups, so a check on either would never refuse there, +`advertising_marketing.first_party.contextual` needs a signal in `gdpr-eu` and +`gdpr-uk`, and the umbrella `functional` is denied in all three, so the call +would never be made. Choosing one is therefore a policy decision rather than a +reading of the taxonomy, which is what `docs/guide/permission-model.md` asks a +declaration to be. The nearest precedent in the stack declares none, because +the built-in and host device modules require no permission and a device module +a module supplies is refused at startup if it declares one +(`crates/trusted-server-core/src/integrations/registry.rs`). A request filter +has no `required_permissions` of its own (`IntegrationRequestFilter` in the +same file), so this one would read `RequestFilterInput::permissions` itself, +which no shipped filter does yet. + +The second is how long an answer is held. The hour was chosen because the +visitor's address was in the evidence and an address can be reassigned. The +evidence now carries no address, so the hour only bounds how long an answer is +kept before the service is asked again, and whether it should be longer is +open. + +**When it fails.** A service that cannot be reached, answers a status that is +not a success, answers a list of errors, answers something that is not JSON, +or answers more than 1 MiB leaves no names. The page is then served as it would +be without the middleware, a warning is logged, and the failure is not held, so +the next navigation with the same evidence asks again. An answer that names no +hints is held like any other, for up to an hour. On Fastly a call that cannot +connect fails after one second, and one that connects waits `timeout_ms` for +the first byte and for each gap between bytes, so while the service is down +every page navigation an entry covers with no held answer waits for those +limits before its page is built. On Axum, Cloudflare and Spin only the +adapter's HTTP client bounds the call. On Axum a service that cannot be +reached fails within the client's 5 second connection limit, a refused +connection fails at once, and a service that accepts the connection and then +stalls holds the call for up to 30 seconds in all. On Cloudflare and Spin the +runtime's own limit bounds it. So while the service is slow or cannot be +reached, each such navigation waits up to that bound. + +**Rollout.** A deployment without `[shared.51degrees]` is unchanged. On every +adapter, adding the table and an entry naming the middleware changes the +pages that entry covers, served to a browser the service +names hints for. The filter leaves state on the request, so core keeps the +request to the origin +path (`IntegrationRequestState` in +`crates/trusted-server-core/src/integrations/registry.rs`). The page is then +never read from a stored template or stored as one, so each reader's copy is +built from the response of the source `[source]` names +(`2026-10-14-source-modules-design.md`), and an HTML response with a body is +sent `private, no-store`. The fetch from the source goes past the host's cache +only where `[creative_opportunities] origin_readthrough_enabled` is on or the +ad stack runs for the request. A browser the service names no hints for leaves +no state and gets its pages as before, although the service is still asked (the +tests script that answer for a Gecko and a WebKit browser). `critical` makes the +browser repeat its first navigation to carry the hints, so that page is built +twice. The license key has to be in the secret store before the deploy that +carries the table, because the settings look it up as they load. An entry's +`path` limits where the service is asked as well as where the headers and the +tag are written, so the effect on pages reaches only the paths an entry naming +the middleware covers. + +## 3. What is not in this change + +- A 51Degrees device module. The names the filter leaves are there for one to + read, and the shared crate's pipeline takes the properties it registers, + and nothing reads them yet. +- Any change to core. The middleware uses the serve phase, the headers step, + the request state and the middleware and shared tables the stack already + has. +- **`timeout_ms` on Axum, Cloudflare and Spin.** Those adapters' HTTP clients + apply no module's time limit, and + `2026-10-17-request-filters-on-every-adapter-design.md` says why that is + left to a change of its own. +- **Asking permission first, or holding answers longer.** The two open + questions under "Open questions for the reviewer" decide those. + +## 4. Tests + +The middleware crate's tests cover the three headers on a document, `critical` +off, a document with no names, what the response already carried being kept, +every line of it when a response sends each header twice, the tag's shape and +the origin it delegates to, a name that is not a token reaching neither, the +request filter against a scripted service for an asset request, for a table +with no entry and for a path an entry's `path` does not cover, none of which +asks the service, for a path that entry covers and for a Chromium, a Gecko and +a WebKit browser, that the filter sends neither the reader's address nor a +cookie, the middleware's own table refusing a key it does not know, a table +with no license key being refused, that no entry means the headers step asks +no middleware, and that one shared table runs the middleware while the stock +loading cache holds its answers. The shared crate's tests cover the table and +its defaults and its refusal of a key it does not know, the license key's +refusals, the call's form and credential, that the evidence is the +`User-Agent` and the `Sec-CH-UA` hints in name order, that no address and no +cookie reaches the request the client builds, that two readers on the same +browser at different addresses make one call, what it holds across instances +and stores, concurrent requests making one call, the refusals it reports, and +the hint list read out of an answer. The Axum adapter's tests plant an unknown +key in the shared table for +the builder that table runs, check that the example template's `[[serve]]` entry +and the guide's table of middleware name `51degrees.client_hints`, and check +that both crates name their maintainers. Each adapter's list test holds the new +row and the hook order. + +Six things have no test. + +1. `crates/middleware/51degrees/src/fixtures/page-change.settings.toml` is + read by no test, and unlike the other modules' it has no input page or + recorded page beside it. +2. Nothing runs the filter through the registry and the publisher path, so no + test shows that a page the filter leaves names for skips the stored + template or is sent `private, no-store`. The tests seed the document state + by hand where core would copy it from the request's state. +3. Nothing shows that `shared.51degrees.license_key` is among the secret + settings a push treats as naming a key, because the Axum adapter's test + lists DataDome's two secrets and no others. +4. No test uncomments the example template's `[shared.51degrees]` and + `[middleware.51degrees.client_hints]` tables to check that they load, as the + test for the Permutive, lockr and Sourcepoint tables does. +5. No adapter test runs the filter, and the filter's tests pass + `permissions: None`, so nothing shows what the filter does with permission + state. +6. The two crates' tests run natively under `cargo test-axum`, and their lints + under `cargo clippy-axum`. No alias runs their tests for an adapter's + WebAssembly target, being `wasm32-wasip1` for Fastly and Spin and + `wasm32-unknown-unknown` for Cloudflare, because the Fastly aliases do not + list them and `cargo test-cloudflare` and `cargo test-spin` test the + adapter crate alone, on the host. + +## Revision record + +| Date | Change | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-14 | First draft. | +| 2026-10-15 | Fixes from review round 1. What the filter runs on and that only the Fastly adapter runs it, the cost to pages, what each navigation sends and the open questions on consent, how each adapter bounds the call, the answers on failure, the settings, what the headers keep, the definition of a page navigation, the untested paths, the related specs and the review log. | +| 2026-10-15 | Fixes from review round 2. The pull request that holds the design documents, the wait for the first call in the guide, and the code comments left for a later change. | +| 2026-10-15 | Code fixes for the call's scope, what it sends and header merging. The permission question stays open, with what was found, and the answer lifetime is a new open question. | +| 2026-10-18 | Every adapter runs the filter, from `2026-10-17-request-filters-on-every-adapter-design.md` and its first review round. The table of parts, the builder, the `timeout_ms` setting, what bounds the call on each adapter, consent, failure, rollout, section 3 and the tests say so. Fixes from review round 3, the wait on Axum for a service that cannot be reached and test item 6. | +| 2026-10-27 | The service's answer is held only where a deployment links a loading cache module, and a stock build, which links none, calls the service on every page navigation an entry covers (`2026-10-12-loading-cache-design.md`, section 7). The table of parts and the declared cache say so. | + +## Review log + +1. **Round 1, 2026-10-15.** A review in a fresh session, against `split/44-client-hints` at `1f6b83d0c`. Issues found, four of them blocking. The four were that the request filter runs wherever the table is present and not only where an entry covers the page, that the cost to existing pages was not stated, that failure handling and the timeout of each adapter were not stated, and that nothing said what each navigation sends to the service or what consent it needs. This revision states each from the code, which also shows that only the Fastly adapter runs request filters, and it corrects the claims about page navigations, the headers and the tests, replaces a related spec that exists in no branch, and corrects the guide, the changelog and the middleware readme. +2. **Round 2, 2026-10-15.** A lighter review in a fresh session, against this page at `126347b00`, of the round 1 fixes and the new text. No blocking issues once the pull request number was corrected. This revision names the pull request that now holds every design document, makes the guide's description of the wait for the first call match the one second connection limit and the limit on each gap between bytes, and lists in section 3 the code comments that still say more than the design does. +3. **Round 3, 2026-10-18.** A review in a fresh session, against `split/52-request-filters` at `17136d04b`, of the change that has every adapter run the filter, made with the request filters design's first round. Issues found, none blocking. The two were that When it fails gave 30 seconds on Axum as the wait for a service that cannot be reached, which fails within the client's 5 second connection limit, and that test item 6 named only the Fastly adapter's target, when no alias runs the two crates' tests for the WebAssembly target of any adapter, all of which now run the filter. Fixed in `e7417c898`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 58f996b3ee..75dc81b190 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -433,6 +433,7 @@ pull_sync_concurrency = 3 # "ad-tag.google.loader", # "ad-tag.google.diagnostics", # "headers.geo", +# "51degrees.client_hints", # "service_version", # ] @@ -526,6 +527,18 @@ pull_sync_concurrency = 3 # store_seconds = 604800 # store_idle_seconds = 2592000 +# The loading cache that holds the 51Degrees service's answers, which the +# 51Degrees modules under [shared.51degrees] ask for once for each request's +# evidence, and how long the copies last, with the same four settings. An +# answer ends after an hour whatever the store lifetime says. A stock build +# links no loading cache module and refuses `module`, as for +# [cache.robots-txt]. +# [cache.51degrees-answers] +# module = "51degrees" +# memory_seconds = 60 +# store_seconds = 604800 +# store_idle_seconds = 86400 + # Operator-controlled cache header policies for static/rehosted assets. Disabled # rules never match, and matcher/policy validation is deferred until a rule is # enabled; IDs must be non-empty and unique. Keep rules disabled unless the @@ -896,6 +909,24 @@ auction_timeout_ms = 500 # script_url = "https://cmp.inmobi.com/choice///choice.js?tag_version=V3" # cmp_id = 10 +# 51Degrees. The table's presence runs the 51Degrees modules, and every one +# of them presents `license_key`, the name of the key's entry in the secret +# store, on its server side calls. `endpoint` is the service's JSON endpoint, +# with any resource key the deployment requires in its path, and `timeout_ms` +# is how long a call waits. The client hints middleware delegates the +# browser's hints to the endpoint's origin, where a [[serve]] entry names +# `51degrees.client_hints`. +# [shared.51degrees] +# endpoint = "https://cloud.51degrees.com/api/v4/json" +# license_key = "fiftyone_degrees_license_key" +# timeout_ms = 500 + +# The client hints middleware's own options. `critical` writes Critical-CH +# with Accept-CH, so a browser retries the first navigation of a session +# carrying its hints, and is on unless set to false. +# [middleware.51degrees.client_hints] +# critical = true + # Permutive DMP. `organization_id` and `workspace_id` are required. # [audience] # module = "permutive"