diff --git a/.cargo/config.toml b/.cargo/config.toml index f366d4683f..7572155938 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,10 +26,10 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index aef0036319..506ec7ee13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `[ad-tag.google] write_loader` writes the loader tag, for a page that carries no loader of its own, through the serve middleware `ad-tag.google.loader`, which a `[[serve]]` entry names and `ts audit` writes where it finds GPT. The tag goes in the head after the bootstrap and asks for the loader on the module's own route. Only a reader whose permissions allow storage on the device (`necessary.operations.storage`) gets it, because GPT loaded from Google's standard address may use the device's storage at any time. That is TCF Purpose 1, which core's server-side auction requires where GDPR applies and which the Google tag module's loaders require. Off by default, because a page that already carries the loader would otherwise load it twice. +- An asset cache module may answer for a file before core asks its origin. Core offers each `GET` for a file that the publisher's origin sends or an asset route fetches, other than one the platform's image optimizer answers, to the module that `[asset-cache]` selects, with what it knows of the reader's request and the request it would send. The module serves a response, which core handles as it would the origin's, reports that its own origin request failed, which core answers as a failed origin request, or passes the file back, and a deployment that selects none has every file asked of its origin. A registration supplies one with `with_asset_cache_module`, and the registry refuses one from a builder that `[asset-cache]` does not select, an `[asset-cache]` that selects two, and one that selects a builder that supplies none, naming that builder. The new `crates/asset-cache/51degrees`, the module `asset-cache.51degrees`, which no stock build links, holds each file as its origin sent it, under what the origin's `Vary` names and for the origin's own lifetime, in two loading caches it declares, `51degrees-assets` for the copies and `51degrees-assets-vary` for each address's record, each selecting its own loading cache module and both sized by `[cache.51degrees-assets]`. It keeps nothing that may not be shared, serves a copy with an `Age` that counts from the origin's, and on the publisher path leaves a document, and every request that carries a cookie, to the origin. A load that fails asking the origin is the answer to every request waiting on it, so a failing origin is asked once. A table that names `none` is refused, as one naming any module no linked crate offers is. - A Google tag module, `[tag] modules = ["google-tag"]` with `[tag.google-tag]`, from `crates/tag/google-tag`, which writes the consent mode defaults, Tag Manager's container script, the gtag.js snippet and the AdSense loader in the head immediately before the script bundle's tag, and Tag Manager's noscript iframe after the opening body tag, each copied from Google's documentation. Every reader gets the consent defaults, and only a reader whose permissions allow storage on the device (`necessary.operations.storage`) gets the loaders and the noscript iframe. The consent defaults come first, the four consent types Google requires for its own services default to denied, the tag waits `consent_defaults.wait_for_update` milliseconds (500 unless set) for a consent management platform's update before it fires with them, and `consent_defaults.region` scopes them to country or region codes, with every other visitor given all four denied. Where the Tag Manager module serves the same tag or container the loader names the first-party path, and a page's own loader for the configured tag is removed for every reader. - A middleware may write markup immediately after the opening body tag, with `body_open_inserts` on its action, once per document, beside the places it had for the head and for straight after the script bundle. - A creative-opportunity format may name `min_viewport_width` and `max_viewport_width`, and a slot with such a format asks for only the formats that fit each band of viewport widths. The slot's page data carries `viewport_sizes`, and the Google ad tag's head bootstrap and the bundle's slot definition apply it as a size mapping, define no slot where the band asks for no size, and keep every format when the mapping fails. The bundle's refresh auction offers a bidder only the sizes of the band the viewport is in for a slot defined with that mapping, and offers a slot the publisher defined its formats as before. A winning bid with no size of its own takes the band's first size, and `ts config ad-templates --details` prints each format's bounds. A slot whose formats name no bound behaves as before. diff --git a/Cargo.lock b/Cargo.lock index d53e3bd58e..b1f58d7d3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6749,6 +6749,26 @@ dependencies = [ "validator", ] +[[package]] +name = "trusted-server-asset-cache-51degrees" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "derive_more", + "edgezero-core", + "error-stack", + "futures", + "hex", + "http", + "log", + "serde_json", + "sha2 0.11.1", + "tokio", + "trusted-server-core", + "web-time", +] + [[package]] name = "trusted-server-auction-aps" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 768a958e7a..08a5524274 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "crates/ads-txt/sellers", "crates/headers/geo", "crates/loading-cache/51degrees", + "crates/asset-cache/51degrees", "crates/tag/google-tag", "crates/tag/google-tag-manager", "crates/testing/testlight", @@ -172,6 +173,7 @@ trusted-server-cmp-inmobi = { path = "crates/cmp/inmobi" } trusted-server-ads-txt-sellers = { path = "crates/ads-txt/sellers" } trusted-server-headers-geo = { path = "crates/headers/geo" } trusted-server-loading-cache-51degrees = { path = "crates/loading-cache/51degrees" } +trusted-server-asset-cache-51degrees = { path = "crates/asset-cache/51degrees" } trusted-server-tag-google-tag = { path = "crates/tag/google-tag" } trusted-server-tag-google-tag-manager = { path = "crates/tag/google-tag-manager" } trusted-server-testing-testlight = { path = "crates/testing/testlight" } diff --git a/crates/asset-cache/51degrees/Cargo.toml b/crates/asset-cache/51degrees/Cargo.toml new file mode 100644 index 0000000000..8ddc0cdf24 --- /dev/null +++ b/crates/asset-cache/51degrees/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "trusted-server-asset-cache-51degrees" +description = "An asset cache module that holds the files the publisher's origin and the asset routes send, as each origin sent them, in two loading caches." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +trusted-server-core = { workspace = true } +async-trait = { workspace = true } +bytes = { workspace = true } +derive_more = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +futures = { workspace = true } +hex = { workspace = true } +http = { workspace = true } +log = { workspace = true } +sha2 = { workspace = true } +web-time = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } +tokio = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. +[package.metadata.maintainers] +owner = "51Degrees" +status = "vendor owned" diff --git a/crates/asset-cache/51degrees/src/entry.rs b/crates/asset-cache/51degrees/src/entry.rs new file mode 100644 index 0000000000..80e23ae384 --- /dev/null +++ b/crates/asset-cache/51degrees/src/entry.rs @@ -0,0 +1,425 @@ +//! What the module keeps for a file, being the origin's response as the +//! origin sent it, and the address it is kept under. +//! +//! A copy holds the headers a reader is served and the body. Every copy is +//! kept under what the origin's own `Vary` names, see `crate::source_cache`. + +use edgezero_core::body::Body as EdgeBody; +use http::{HeaderName, HeaderValue, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; + +/// Version of what a kept copy holds and how it is written. Bump on any change +/// to either, so a copy written by another build is never read. +pub const ASSET_SCHEMA_VERSION: u32 = 1; + +/// The most bytes a kept copy's body holds. A larger response is served as it +/// streams from the origin, because every copy is held whole, in memory and in +/// the shared store. +pub const MAX_ASSET_COPY_BYTES: usize = 1024 * 1024; + +/// Response headers never kept with a copy, being those describing one +/// connection, one reader's cookies, or the length and age, which are set again +/// for each reader. +const NOT_KEPT: &[&str] = &[ + "age", + "connection", + "content-length", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "set-cookie", + "te", + "trailer", + "transfer-encoding", + "upgrade", +]; + +/// Inputs that select one file's address. +/// +/// What the origin's response varies on is not here, because it is read from +/// the address's record once the origin has said it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AssetCacheKey { + /// The URL asked of the origin, with its query. + pub url: String, + /// The host the reader asked, so two hosts that serve one publisher each + /// keep a copy. + pub request_host: String, + /// The scheme the reader asked over. + pub request_scheme: String, + /// What names the origin beyond the URL, as core reports it. + pub origin_identity: String, +} + +impl AssetCacheKey { + /// A fixed-size key naming no part of the request. + /// + /// Each input is written with its length before it, so no two sets of + /// inputs run together into the same key. + #[must_use] + pub fn to_cache_key(&self) -> String { + use crate::hash::hash_length_prefixed as push; + + let mut hasher = Sha256::new(); + push(&mut hasher, b"ts-asset-cache"); + push(&mut hasher, &ASSET_SCHEMA_VERSION.to_be_bytes()); + push(&mut hasher, self.request_scheme.as_bytes()); + push(&mut hasher, self.request_host.as_bytes()); + push(&mut hasher, self.origin_identity.as_bytes()); + push(&mut hasher, self.url.as_bytes()); + format!( + "ts-asset-cache-v{ASSET_SCHEMA_VERSION}-{}", + hex::encode(hasher.finalize()) + ) + } +} + +/// One kept response. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AssetCopy { + /// The headers a reader is served, as the origin sent them. + pub headers: Vec<(String, String)>, + /// Whether the body is kept without its `Content-Encoding`. Every copy is + /// kept as the origin sent it, so this is false. + pub decoded: bool, + /// When the copy was made, in seconds since the Unix epoch. + pub made_at: u64, + /// The response's age when the copy was made, in seconds. + pub age_at_made: u64, + /// The body. + pub body: Vec, +} + +impl AssetCopy { + /// The headers of `response_headers` a copy keeps, or `None` when one + /// cannot be written down, as a value that is not visible text cannot. + /// + /// `Content-Encoding` is left out of a decoded copy, which holds identity + /// bytes. + #[must_use] + pub fn kept_headers( + response_headers: &http::HeaderMap, + decoded: bool, + ) -> Option> { + let mut kept = Vec::new(); + for (name, value) in response_headers { + if NOT_KEPT.contains(&name.as_str()) || (decoded && name == header::CONTENT_ENCODING) { + continue; + } + let value = value.to_str().ok()?; + kept.push((name.as_str().to_owned(), value.to_owned())); + } + Some(kept) + } + + /// The response this copy is served as, `now` seconds since the Unix + /// epoch, its `Age` counting the time it has been kept. + pub fn into_response(self, now: u64) -> Response { + let mut response = Response::new(EdgeBody::empty()); + *response.status_mut() = StatusCode::OK; + for (name, value) in &self.headers { + // Both were checked when the copy was read. + if let (Ok(name), Ok(value)) = ( + HeaderName::from_bytes(name.as_bytes()), + HeaderValue::from_str(value), + ) { + response.headers_mut().append(name, value); + } + } + let age = self + .age_at_made + .saturating_add(now.saturating_sub(self.made_at)); + response + .headers_mut() + .insert(header::AGE, HeaderValue::from(age)); + response.headers_mut().insert( + header::CONTENT_LENGTH, + HeaderValue::from(self.body.len() as u64), + ); + *response.body_mut() = EdgeBody::from(self.body); + response + } +} + +/// What the module holds for one request's values. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AssetEntry { + /// A response every request with the same values may be served. + Copy(AssetCopy), + /// The origin's response for these values may not be shared, so each + /// request fetches its own. Held briefly, so the requests that follow do + /// not wait on a load that ends the same way. + NotShareable, +} + +/// The first byte of a held [`AssetEntry`], which says its kind. +const ENTRY_NOT_SHAREABLE: u8 = 0; +const ENTRY_COPY: u8 = 1; + +/// The bytes of a copy's description length, a little-endian `u32`. +const DESCRIPTION_LENGTH_BYTES: usize = size_of::(); + +/// The bytes a held copy starts with, its kind byte and its description +/// length. +const COPY_HEAD_BYTES: usize = 1 + DESCRIPTION_LENGTH_BYTES; + +/// Why a held asset entry could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum AssetCacheMiss { + /// Written by a build keeping another format. + #[display("the kept asset was written in another format")] + OtherFormat, + /// Unreadable, or shorter than it says it is. + #[display("the kept asset is unreadable")] + Unreadable, +} + +impl core::error::Error for AssetCacheMiss {} + +impl AssetEntry { + /// The bytes the cache holds for this entry, being a kind byte, then for a + /// copy the length of its description as a little-endian `u32`, the + /// description, one line per field, and the body. + #[must_use] + pub fn encode(&self) -> Vec { + match self { + Self::NotShareable => vec![ENTRY_NOT_SHAREABLE], + Self::Copy(copy) => { + let mut description = format!( + "v={ASSET_SCHEMA_VERSION}\nd={}\nt={}\na={}\nlen={}", + u8::from(copy.decoded), + copy.made_at, + copy.age_at_made, + copy.body.len() + ); + for (name, value) in ©.headers { + description.push_str(&format!("\nh={name}:{value}")); + } + let length = u32::try_from(description.len()).unwrap_or(u32::MAX); + let mut bytes = + Vec::with_capacity(COPY_HEAD_BYTES + description.len() + copy.body.len()); + bytes.push(ENTRY_COPY); + bytes.extend_from_slice(&length.to_le_bytes()); + bytes.extend_from_slice(description.as_bytes()); + bytes.extend_from_slice(©.body); + bytes + } + } + } + + /// The entry `bytes` hold. + /// + /// # Errors + /// + /// When the bytes are not an entry this build wrote. + pub fn decode(bytes: &[u8]) -> Result { + match bytes.split_first() { + Some((&ENTRY_NOT_SHAREABLE, [])) => Ok(Self::NotShareable), + Some((&ENTRY_COPY, rest)) => { + let (length, rest) = rest + .split_first_chunk::() + .ok_or(AssetCacheMiss::Unreadable)?; + let length = usize::try_from(u32::from_le_bytes(*length)) + .map_err(|_| AssetCacheMiss::Unreadable)?; + if rest.len() < length { + return Err(AssetCacheMiss::Unreadable); + } + let (description, body) = rest.split_at(length); + let description = + std::str::from_utf8(description).map_err(|_| AssetCacheMiss::Unreadable)?; + decode_copy(description, body).map(Self::Copy) + } + _ => Err(AssetCacheMiss::Unreadable), + } + } +} + +fn decode_copy(description: &str, body: &[u8]) -> Result { + let unreadable = AssetCacheMiss::Unreadable; + let mut version = None; + let mut decoded = None; + let mut made_at = None; + let mut age_at_made = None; + let mut length = None; + let mut headers = Vec::new(); + for line in description.split('\n') { + let (field, value) = line.split_once('=').ok_or(unreadable)?; + let once = |held: Option| held.is_none().then_some(()).ok_or(unreadable); + match field { + "v" => { + once(version)?; + version = Some(value.parse::().map_err(|_| unreadable)?); + } + "d" => { + decoded = Some(match value { + "0" if decoded.is_none() => false, + "1" if decoded.is_none() => true, + _ => return Err(unreadable), + }); + } + "t" => { + once(made_at)?; + made_at = Some(value.parse().map_err(|_| unreadable)?); + } + "a" => { + once(age_at_made)?; + age_at_made = Some(value.parse().map_err(|_| unreadable)?); + } + "len" => { + once(length)?; + length = Some(value.parse().map_err(|_| unreadable)?); + } + "h" => { + let (name, header_value) = value.split_once(':').ok_or(unreadable)?; + let parsed = HeaderName::from_bytes(name.as_bytes()).map_err(|_| unreadable)?; + if parsed.as_str() != name || NOT_KEPT.contains(&name) { + return Err(unreadable); + } + HeaderValue::from_str(header_value).map_err(|_| unreadable)?; + headers.push((name.to_owned(), header_value.to_owned())); + } + _ => return Err(unreadable), + } + } + if version.ok_or(unreadable)? != u64::from(ASSET_SCHEMA_VERSION) { + return Err(AssetCacheMiss::OtherFormat); + } + if length.ok_or(unreadable)? != body.len() as u64 { + return Err(unreadable); + } + Ok(AssetCopy { + headers, + decoded: decoded.ok_or(unreadable)?, + made_at: made_at.ok_or(unreadable)?, + age_at_made: age_at_made.ok_or(unreadable)?, + body: body.to_vec(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key() -> AssetCacheKey { + AssetCacheKey { + url: "https://origin.example.com/assets/site.css".to_owned(), + request_host: "www.example.com".to_owned(), + request_scheme: "https".to_owned(), + origin_identity: "https://origin.example.com\0origin.example.com".to_owned(), + } + } + + fn copy() -> AssetCopy { + AssetCopy { + headers: vec![ + ("content-type".to_owned(), "text/css".to_owned()), + ("vary".to_owned(), "Accept-Encoding".to_owned()), + ], + decoded: true, + made_at: 1_000, + age_at_made: 5, + body: b"body { color: red }".to_vec(), + } + } + + #[test] + fn every_input_changes_the_key_and_none_appears_in_it() { + let base = key().to_cache_key(); + let changes: [fn(&mut AssetCacheKey); 4] = [ + |key| key.url.push_str("?v=2"), + |key| key.request_host = "other.example.com".to_owned(), + |key| key.request_scheme = "http".to_owned(), + |key| key.origin_identity.push('x'), + ]; + for change in changes { + let mut changed = key(); + change(&mut changed); + assert_ne!(changed.to_cache_key(), base); + } + assert!(base.starts_with("ts-asset-cache-v1-")); + for named in ["example.com", "site.css"] { + assert!(!base.contains(named), "the key names `{named}`: {base}"); + } + } + + #[test] + fn an_entry_reads_back_as_written() { + for entry in [AssetEntry::Copy(copy()), AssetEntry::NotShareable] { + assert_eq!(AssetEntry::decode(&entry.encode()), Ok(entry)); + } + } + + #[test] + fn a_short_foreign_or_other_format_entry_is_not_read() { + let bytes = AssetEntry::Copy(copy()).encode(); + assert_eq!( + AssetEntry::decode(&bytes[..bytes.len() - 1]), + Err(AssetCacheMiss::Unreadable) + ); + for foreign in [&b""[..], b"\x07", b"\x01\xff\xff", b"\x00extra"] { + assert!(AssetEntry::decode(foreign).is_err(), "{foreign:?}"); + } + let other_version = String::from_utf8(bytes.clone()) + .expect("the test copy is text") + .replacen("v=1", "v=9", 1); + assert_eq!( + AssetEntry::decode(other_version.as_bytes()), + Err(AssetCacheMiss::OtherFormat) + ); + } + + #[test] + fn a_copy_keeps_no_connection_cookie_or_length_header() { + let mut response = http::HeaderMap::new(); + for (name, value) in [ + ("content-type", "image/png"), + ("content-encoding", "gzip"), + ("set-cookie", "a=b"), + ("content-length", "10"), + ("age", "3"), + ("connection", "keep-alive"), + ("cache-control", "public, max-age=60"), + ] { + response.append( + HeaderName::from_static(name), + HeaderValue::from_static(value), + ); + } + + let raw = AssetCopy::kept_headers(&response, false).expect("should keep the headers"); + let decoded = AssetCopy::kept_headers(&response, true).expect("should keep the headers"); + + let names = |kept: &[(String, String)]| -> Vec { + kept.iter().map(|(name, _)| name.clone()).collect() + }; + assert_eq!( + names(&raw), + ["content-type", "content-encoding", "cache-control"] + ); + assert_eq!( + names(&decoded), + ["content-type", "cache-control"], + "a decoded copy's encoding is set for each reader" + ); + } + + #[test] + fn a_served_copy_counts_its_age_from_when_it_was_made() { + let response = copy().into_response(1_030); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers().get(header::AGE), + Some(&HeaderValue::from_static("35")) + ); + assert_eq!( + response.headers().get(header::CONTENT_LENGTH), + Some(&HeaderValue::from_static("19")) + ); + assert_eq!( + response.headers().get(header::CONTENT_TYPE), + Some(&HeaderValue::from_static("text/css")) + ); + } +} diff --git a/crates/asset-cache/51degrees/src/hash.rs b/crates/asset-cache/51degrees/src/hash.rs new file mode 100644 index 0000000000..3db23128dc --- /dev/null +++ b/crates/asset-cache/51degrees/src/hash.rs @@ -0,0 +1,10 @@ +//! The hashing both of the module's keys use. + +use sha2::Digest as _; + +/// Writes `part` into `hasher` with its length before it, so no two lists of +/// parts run together into the same digest. +pub(crate) fn hash_length_prefixed(hasher: &mut sha2::Sha256, part: &[u8]) { + hasher.update((part.len() as u64).to_be_bytes()); + hasher.update(part); +} diff --git a/crates/asset-cache/51degrees/src/lib.rs b/crates/asset-cache/51degrees/src/lib.rs new file mode 100644 index 0000000000..feb75aef81 --- /dev/null +++ b/crates/asset-cache/51degrees/src/lib.rs @@ -0,0 +1,221 @@ +//! An asset cache module that holds the files the publisher's origin sends +//! that are not pages, and the answers of the asset routes, as each origin +//! sent them. +//! +//! `[asset-cache] module = "51degrees"` selects it, and core then offers it +//! each `GET` for such a file before asking the file's origin, see +//! [`trusted_server_core::asset_cache`]. A file's copies are held in +//! `51degrees-assets` under what the origin's `Vary` names, which the module +//! records for each address in `51degrees-assets-vary`. Both are loading +//! caches the module declares, so each takes a loading cache module the way +//! every declared cache does, and holds nothing in a build that links none. + +pub(crate) mod entry; +mod hash; +mod load; +pub(crate) mod source_cache; + +use std::sync::Arc; + +use error_stack::Report; +use trusted_server_core::asset_cache::{AssetAnswer, AssetCacheModule, AssetFile, AssetVia}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{IntegrationBuilder, IntegrationRegistration}; +use trusted_server_core::platform::{CacheLifetimes, CacheSpec, RuntimeServices}; +use trusted_server_core::settings::{LoadingCacheConfig, Settings}; + +use crate::entry::AssetCacheKey; +use crate::source_cache::SourceCache; + +/// The name of this module, from this crate's folder. +pub const MODULE: &str = "asset-cache.51degrees"; + +/// The cache that holds the copies, whose lifetimes `[cache.51degrees-assets]` +/// sets. +pub const ASSETS_CACHE: &str = "51degrees-assets"; + +/// The cache that holds each address's record of what its responses vary on, +/// with the lifetimes `[cache.51degrees-assets]` sets for the copies. +pub const ASSETS_VARY_CACHE: &str = "51degrees-assets-vary"; + +const INTEGRATION_ID: &str = "asset_cache_51degrees"; + +/// The lifetimes `[cache.51degrees-assets]` leaves unset. A file's own +/// lifetime ends each copy sooner than the store lifetime. +const ASSET_DEFAULT_LIFETIMES: CacheLifetimes = CacheLifetimes { + memory: std::time::Duration::from_secs(5), + store: std::time::Duration::from_secs(60 * 60), + store_idle: std::time::Duration::from_secs(24 * 60 * 60), +}; + +/// The most records the memory layer of `51degrees-assets-vary` holds. +const MEMORY_RECORDS: usize = 256; + +/// The most copies the memory layer of `51degrees-assets` holds. +const MEMORY_COPIES: usize = 32; + +/// The builder a deployment hands to an adapter, which the registry runs +/// when `[asset-cache]` selects [`MODULE`]. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(INTEGRATION_ID, env!("CARGO_PKG_NAME"), register, validate) + .with_module_name(MODULE) + .with_cache_features(&[ASSETS_CACHE, ASSETS_VARY_CACHE]) +} + +/// Reports whether a section selects the module. +/// +/// The module reads no table of its own, so it refuses an +/// `[asset-cache.51degrees]` table, an empty one included. It refuses a +/// lifetime in `[cache.51degrees-assets-vary]` whether or not it is selected. +fn validate(settings: &Settings) -> Result> { + refuse_record_lifetimes(settings)?; + let Some((section, written)) = settings.module_selection(MODULE) else { + return Ok(false); + }; + let has_table = settings + .module_sections() + .any(|(name, modules)| name == section && modules.settings_of(written).is_some()); + if has_table { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{section}.{written}] is a table the module does not read, because it \ + takes no setting of its own, so remove the table. The lifetimes of its \ + caches are set in [cache.{ASSETS_CACHE}]" + ), + })); + } + Ok(true) +} + +/// Refuses each lifetime `[cache.51degrees-assets-vary]` sets, because the +/// records take the lifetimes of `[cache.51degrees-assets]` and nothing reads +/// the table's own. +fn refuse_record_lifetimes(settings: &Settings) -> Result<(), Report> { + let Some(table) = settings.cache.declared.get(ASSETS_VARY_CACHE) else { + return Ok(()); + }; + // Every field is named, so a setting added to the table does not build + // here until this check reads or refuses it. + let LoadingCacheConfig { + module: _, + memory_seconds, + store_seconds, + store_idle_seconds, + } = table; + let set: Vec = [ + ("memory_seconds", memory_seconds), + ("store_seconds", store_seconds), + ("store_idle_seconds", store_idle_seconds), + ] + .into_iter() + .filter(|(_, value)| value.is_some()) + .map(|(name, _)| format!("`{name}`")) + .collect(); + let named = match set.split_last() { + None => return Ok(()), + Some((last, [])) => last.clone(), + Some((last, rest)) => format!("{} and {last}", rest.join(", ")), + }; + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[cache.{ASSETS_VARY_CACHE}] sets {named}, which {MODULE} does not read, because \ + its records take the lifetimes of [cache.{ASSETS_CACHE}]. Set the lifetimes \ + there, as this table takes only `module`" + ), + })) +} + +/// Supplies the module when `[asset-cache]` selects it. +fn register( + settings: &Settings, +) -> Result, Report> { + if !validate(settings)? { + return Ok(None); + } + Ok(Some( + IntegrationRegistration::builder(INTEGRATION_ID) + .without_js() + .with_asset_cache_module(MODULE, Arc::new(AssetCache::from_settings(settings))) + .build(), + )) +} + +/// The module, which answers for a file from the copies it holds and loads +/// the file from its origin when none serves. +#[derive(Debug)] +pub struct AssetCache { + caches: SourceCache, +} + +impl AssetCache { + /// The module with both of its caches given the lifetimes + /// `[cache.51degrees-assets]` sets, and the defaults for any it leaves + /// unset. + #[must_use] + pub fn from_settings(settings: &Settings) -> Self { + let lifetimes = settings + .cache + .declared + .get(ASSETS_CACHE) + .map_or(ASSET_DEFAULT_LIFETIMES, |config| { + config.lifetimes(ASSET_DEFAULT_LIFETIMES) + }); + let spec = |name, memory_entries| CacheSpec { + name, + lifetimes, + memory_entries, + }; + Self { + caches: SourceCache { + records: spec(ASSETS_VARY_CACHE, MEMORY_RECORDS), + copies: spec(ASSETS_CACHE, MEMORY_COPIES), + }, + } + } +} + +#[async_trait::async_trait(?Send)] +impl AssetCacheModule for AssetCache { + async fn answer(&self, services: &RuntimeServices, file: AssetFile<'_>) -> AssetAnswer { + if !may_use_a_copy(services, &file) { + return AssetAnswer::AskOrigin; + } + let key = AssetCacheKey { + url: file.origin_url.to_owned(), + request_host: file.host.to_owned(), + request_scheme: file.scheme.to_owned(), + origin_identity: file.origin_identity.to_owned(), + }; + match load::asset_from_cache(services, &self.caches, &key, file.origin).await { + Some(Ok(response)) => AssetAnswer::Serve(response), + Some(Err(report)) => AssetAnswer::OriginFailed(report), + None => AssetAnswer::AskOrigin, + } + } +} + +/// Whether a copy may answer for `file`, which core offers only as a `GET`. +/// +/// On the publisher path the origin receives the reader's cookies, and +/// nothing shows whether they shaped its answer, so a request carrying a +/// cookie is the origin's, as a document is. A route sends its origin no +/// cookie. A record cache that keeps nothing would run the record's load, +/// and so fetch the origin, for every request, so both caches must keep +/// values. +fn may_use_a_copy(services: &RuntimeServices, file: &AssetFile<'_>) -> bool { + let reader_allows = match file.via { + AssetVia::Publisher => !file.is_document && !file.carries_cookie, + AssetVia::Route => true, + }; + reader_allows + && !file.host.is_empty() + && !file.carries_authorization + && !file.asks_for_origin + && [ASSETS_CACHE, ASSETS_VARY_CACHE] + .iter() + .all(|name| services.loading_cache_for(name).keeps_values()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/asset-cache/51degrees/src/load.rs b/crates/asset-cache/51degrees/src/load.rs new file mode 100644 index 0000000000..73fb24e20c --- /dev/null +++ b/crates/asset-cache/51degrees/src/load.rs @@ -0,0 +1,427 @@ +//! How the module answers for a file from its caches, and loads the file +//! from its origin when no copy serves. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::{Rc, Weak}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, SystemTime}; + +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use futures::StreamExt as _; +use http::{Response, StatusCode, header}; +use trusted_server_core::asset_cache::OriginRequest; +use trusted_server_core::platform::{LoadError, PlatformError, RuntimeServices}; +use trusted_server_core::publisher::{KeepRefusal, origin_response_may_be_kept, origin_shared_ttl}; +use web_time::Instant; + +use crate::entry::{AssetCacheKey, AssetCopy, AssetEntry, MAX_ASSET_COPY_BYTES}; +use crate::source_cache::{ + Fetch, Fetched, Found, Lookup, SourceAddress, SourceCache, SourceVary, get_or_fetch, +}; + +/// How long a file the origin says may not be shared is held as such, so +/// the requests that follow fetch their own rather than waiting on a load. +const ASSET_NOT_SHAREABLE_LIFETIME: Duration = Duration::from_secs(60); + +/// The address one file's copies are held under, made of the publisher's +/// prefix the request's services carry, so two publishers never share one, +/// and the file's own key. +fn asset_address(services: &RuntimeServices, key: &AssetCacheKey) -> SourceAddress { + SourceAddress::new(services.server_key_prefix().clone(), key.to_cache_key()) +} + +/// The answer for the file `key` names, or `None` when core is to ask the +/// origin. The answer is a kept copy, the response this request's own load +/// fetched and may not share, or the failure of the load this request ran or +/// waited on, which core treats as its own origin request failing, so the +/// origin is not asked again. +pub(crate) async fn asset_from_cache( + services: &RuntimeServices, + cache: &SourceCache, + key: &AssetCacheKey, + origin: OriginRequest, +) -> Option, Report>> { + let address = asset_address(services, key); + let hand_back = HandBack::new(); + let load = Arc::new(AssetLoad { + services: services.clone().without_connection(), + origin, + max_lifetime: cache.copies.lifetimes.store, + }); + let fetch = asset_fetch(Arc::clone(&load), hand_back.id); + let not_shareable = Bytes::from(AssetEntry::NotShareable.encode()); + let found = get_or_fetch( + &Lookup { + services, + cache, + address: &address, + headers: load.origin.headers(), + declared: None, + not_shareable: ¬_shareable, + }, + fetch, + ) + .await; + match found { + Ok(Found::Copy { value, key }) => match AssetEntry::decode(&value) { + Ok(AssetEntry::Copy(copy)) => Some(Ok(copy.into_response(unix_seconds()))), + // This request's own load fetched the file, so it is served from + // that fetch rather than a second one. + Ok(AssetEntry::NotShareable) => hand_back.take().map(Ok), + Err(miss) => { + log::warn!("asset_cache invalid entry: {miss}; removing it"); + services + .loading_cache_for(cache.copies.name) + .remove(services, &cache.copies, &key) + .await; + None + } + }, + Ok(Found::NoCopies) => hand_back.take().map(Ok), + // The load failed asking the origin, and every request waiting on it + // is given that failure, so none of them asks the origin again. + Err(error) => { + log::warn!("asset_cache load failed: {error}"); + Some(Err( + Report::new(error).change_context(PlatformError::HttpClient) + )) + } + } +} + +/// Why a file's response is not kept. +#[derive(Debug, derive_more::Display)] +enum AssetCacheBypassReason { + /// A page is kept as a template, by the template cache, or not at all. + #[display("the response is HTML, which is not kept as an asset")] + Html, + /// The body is larger than a copy holds. + #[display("the response is larger than a kept asset holds")] + TooLarge, + /// The body outgrew a copy, or failed, while it was read. + #[display("the response could not be read whole")] + NotReadWhole, + /// A header cannot be written down to be served again. + #[display("a response header cannot be kept")] + UnkeptHeader, + /// The response varies on every request header. + #[display("origin response varies on *")] + VaryWildcard, + /// The response varies on the reader's cookies. + #[display("origin response varies on Cookie")] + VaryCookie, + /// The response's `Vary` cannot be read. + #[display("origin response Vary cannot be read")] + VaryUnreadable, + /// The origin's status, cookies or cache policy do not allow a shared + /// copy, or the policy cannot be read. + #[display("{_0}")] + Policy(KeepRefusal), +} + +/// Why a response that varies this way is served to no other request. +fn vary_bypass_reason(vary: &SourceVary) -> Option { + match vary { + SourceVary::Headers(_) => None, + SourceVary::Any => Some(AssetCacheBypassReason::VaryWildcard), + SourceVary::Cookie => Some(AssetCacheBypassReason::VaryCookie), + SourceVary::Unreadable => Some(AssetCacheBypassReason::VaryUnreadable), + } +} + +/// How long a file's response may be kept, from its own cache policy and +/// capped at `max_lifetime`, or why it is not kept at all. +fn asset_cache_ttl( + status: StatusCode, + content_type: &str, + headers: &http::HeaderMap, + max_lifetime: Duration, +) -> Result { + origin_response_may_be_kept(status, headers).map_err(AssetCacheBypassReason::Policy)?; + if is_html_content_type(content_type) { + return Err(AssetCacheBypassReason::Html); + } + origin_shared_ttl(headers, max_lifetime).map_err(AssetCacheBypassReason::Policy) +} + +/// Seconds since the Unix epoch, from the wasm-safe clock. +fn unix_seconds() -> u64 { + std_system_time_now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |since| since.as_secs()) +} + +/// The current time as a [`std::time::SystemTime`] without calling +/// `std::time::SystemTime::now()`, which panics on `wasm32-unknown-unknown`. +/// It reads the [`web_time`] clock, which on native and `wasm32-wasip1` is the +/// std clock. +fn std_system_time_now() -> SystemTime { + std::time::UNIX_EPOCH + + web_time::SystemTime::now() + .duration_since(web_time::UNIX_EPOCH) + .unwrap_or_default() +} + +fn is_html_content_type(content_type: &str) -> bool { + content_type_contains_ascii_case_insensitive(content_type, "text/html") +} + +fn content_type_contains_ascii_case_insensitive(content_type: &str, needle: &str) -> bool { + content_type.to_ascii_lowercase().contains(needle) +} + +fn single_header_value( + headers: &http::HeaderMap, + name: header::HeaderName, +) -> Result, KeepRefusal> { + let mut values = headers.get_all(name).iter(); + let Some(first) = values.next() else { + return Ok(None); + }; + if values.next().is_some() { + return Err(KeepRefusal::MalformedPolicy); + } + first + .to_str() + .map(Some) + .map_err(|_| KeepRefusal::MalformedPolicy) +} + +fn parse_delta_seconds(value: &str) -> Result { + let value = value.trim(); + let quoted_at_start = value.starts_with('"'); + let quoted_at_end = value.ends_with('"'); + let digits = match (quoted_at_start, quoted_at_end) { + (true, true) if value.len() >= 2 => &value[1..value.len() - 1], + (false, false) => value, + _ => return Err(KeepRefusal::MalformedPolicy), + }; + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(KeepRefusal::MalformedPolicy); + } + digits + .parse::() + .map_err(|_| KeepRefusal::MalformedPolicy) +} + +type HandBackSlot = RefCell>>; + +/// The slot of each load on this thread whose request is still waiting, by +/// the load's id. +type HandBackSlots = RefCell>>; + +thread_local! { + static HANDED_BACK: HandBackSlots = RefCell::new(HashMap::new()); +} + +static NEXT_LOAD: AtomicU64 = AtomicU64::new(0); + +/// Where a load leaves a response the request that started it is served, +/// because the origin answered with something no other request may be +/// served, so that request is served from the one fetch rather than a +/// second one. The slot lives as long as the request's `HandBack`, on the +/// thread that runs the request. +struct HandBack { + id: u64, + slot: Rc, +} + +impl HandBack { + fn new() -> Self { + let id = NEXT_LOAD.fetch_add(1, Ordering::Relaxed); + let slot = Rc::new(RefCell::new(None)); + HANDED_BACK.with(|handed| { + handed.borrow_mut().insert(id, Rc::downgrade(&slot)); + }); + Self { id, slot } + } + + fn take(&self) -> Option> { + self.slot.borrow_mut().take() + } +} + +impl Drop for HandBack { + fn drop(&mut self) { + HANDED_BACK.with(|handed| { + handed.borrow_mut().remove(&self.id); + }); + } +} + +fn hand_back(id: u64, response: Response) { + let slot = HANDED_BACK.with(|handed| handed.borrow().get(&id).and_then(Weak::upgrade)); + if let Some(slot) = slot { + *slot.borrow_mut() = Some(response); + } +} + +/// What a load owns to fetch one file, because a loader owns everything it +/// uses. +struct AssetLoad { + /// The request's services without its connection, because one load + /// serves every request waiting on it. + services: RuntimeServices, + /// The request core would send to the origin. + origin: OriginRequest, + /// The longest a copy may be kept, whatever the origin asks. + max_lifetime: Duration, +} + +/// What one fetch of a file made. +struct AssetLoaded { + /// What the origin's response varies on. + vary: SourceVary, + /// The copy with the time it may be used for, or `None` when the response + /// may not be shared. + copy: Option<(AssetCopy, Duration)>, +} + +/// The module's fetch for one request, which fetches the file and, when it +/// may be shared, keeps it as the origin sent it. +/// +/// A file that may not be shared is held as [`AssetEntry::NotShareable`] for +/// [`ASSET_NOT_SHAREABLE_LIFETIME`], and handed back to the request whose +/// load fetched it through `hand_back_id`, see [`HandBack`]. +fn asset_fetch(load: Arc, hand_back_id: u64) -> Fetch { + Arc::new(move || { + let load = Arc::clone(&load); + Box::pin(async move { + let AssetLoaded { vary, copy } = load_asset(&load, hand_back_id).await?; + let (entry, lifetime) = match copy { + Some((copy, lifetime)) => (AssetEntry::Copy(copy), lifetime), + None => (AssetEntry::NotShareable, ASSET_NOT_SHAREABLE_LIFETIME), + }; + Ok(Fetched { + vary, + copy: entry.encode().into(), + lifetime, + }) + }) + }) +} + +/// `body` read whole when it holds at most `limit` bytes, or else the body to +/// serve in its place, being what was read so far and then the rest. +async fn read_whole_or_hand_back(body: EdgeBody, limit: usize) -> Result { + let mut stream = match body { + EdgeBody::Once(bytes) if bytes.len() > limit => return Err(EdgeBody::Once(bytes)), + EdgeBody::Once(bytes) => return Ok(bytes), + EdgeBody::Stream(stream) => stream, + }; + let mut read = Vec::new(); + while let Some(chunk) = stream.next().await { + let failed = match chunk { + Ok(chunk) => { + read.extend_from_slice(&chunk); + if read.len() <= limit { + continue; + } + None + } + Err(error) => Some(Err(error)), + }; + let head = std::iter::once(Ok(Bytes::from(read))).chain(failed); + return Err(EdgeBody::Stream( + futures::stream::iter(head).chain(stream).boxed_local(), + )); + } + Ok(read.into()) +} + +/// Fetches a file and keeps it as the origin sent it, with the time it may +/// be used for, or no copy when it may not be shared. +/// +/// # Errors +/// +/// When the origin cannot be reached. +async fn load_asset(load: &AssetLoad, hand_back_id: u64) -> Result { + let response = load + .services + .http_client() + .send(load.origin.platform_request()) + .await + .map_err(|report| LoadError::new(format!("the origin could not be reached: {report:?}")))? + .response; + let made = Instant::now(); + let (parts, body) = response.into_parts(); + let content_type = parts + .headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let vary = SourceVary::of(&parts.headers); + let not_shareable = |vary, parts, body| { + hand_back(hand_back_id, Response::from_parts(parts, body)); + Ok(AssetLoaded { vary, copy: None }) + }; + if let Some(reason) = vary_bypass_reason(&vary) { + log::debug!("asset_cache bypass: {reason}"); + return not_shareable(vary, parts, body); + } + let ttl = match asset_cache_ttl( + parts.status, + &content_type, + &parts.headers, + load.max_lifetime, + ) { + Ok(ttl) => ttl, + Err(reason) => { + log::debug!("asset_cache bypass: {reason}"); + return not_shareable(vary, parts, body); + } + }; + let declared_length = parts + .headers + .get(header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()); + if declared_length.is_some_and(|length| length > MAX_ASSET_COPY_BYTES as u64) { + log::debug!("asset_cache bypass: {}", AssetCacheBypassReason::TooLarge); + return not_shareable(vary, parts, body); + } + let Some(headers) = AssetCopy::kept_headers(&parts.headers, false) else { + log::debug!( + "asset_cache bypass: {}", + AssetCacheBypassReason::UnkeptHeader + ); + return not_shareable(vary, parts, body); + }; + let age_at_made = single_header_value(&parts.headers, header::AGE) + .ok() + .flatten() + .and_then(|value| parse_delta_seconds(value).ok()) + .unwrap_or(0); + let raw = match read_whole_or_hand_back(body, MAX_ASSET_COPY_BYTES).await { + Ok(raw) => raw, + Err(body) => { + log::debug!( + "asset_cache bypass: {}", + AssetCacheBypassReason::NotReadWhole + ); + return not_shareable(vary, parts, body); + } + }; + let lifetime = ttl.saturating_sub(made.elapsed()); + if !lifetime.is_zero() { + log::debug!("asset_cache stored {} bytes", raw.len()); + } + Ok(AssetLoaded { + vary, + copy: Some(( + AssetCopy { + headers, + decoded: false, + made_at: unix_seconds(), + age_at_made, + body: raw.to_vec(), + }, + lifetime, + )), + }) +} diff --git a/crates/asset-cache/51degrees/src/source_cache.rs b/crates/asset-cache/51degrees/src/source_cache.rs new file mode 100644 index 0000000000..6f46fda851 --- /dev/null +++ b/crates/asset-cache/51degrees/src/source_cache.rs @@ -0,0 +1,744 @@ +//! The cache that holds a source's responses, keyed the way the source says +//! its responses vary. +//! +//! A request's address is what it asks the source for. The first response for +//! an address records the request headers its `Vary` names, and that record is +//! held in the loading cache like any other value. A later request for the +//! address reads the record and is served the copy kept under its own values +//! for those headers, as RFC 9111 section 4.1 describes. +//! +//! Requests for an address no record covers share one load. The request that +//! leads it fetches the source and writes its copy under its own values before +//! the record is published, so a waiting request with the same values is +//! served that copy, and one with other values loads its own, sharing that +//! load with every request waiting on the same values. The request whose fetch +//! ran is served what it fetched, whatever the cache kept, so no request +//! fetches the source twice. +//! +//! A response whose `Vary` is `*`, names `Cookie`, or cannot be read can be +//! served to no other request, so its record says no copy is kept, and every +//! request for the address fetches its own response until the record ends. + +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use http::HeaderMap; +use http::header::{self, HeaderName}; +use sha2::{Digest as _, Sha256}; + +use trusted_server_core::platform::{ + CacheSpec, LoadError, LoadedValue, Loader, RuntimeServices, ServerKeyPrefix, StoreKey, +}; + +use crate::hash::hash_length_prefixed as push; + +/// What a source's response says it varies on, read from its `Vary`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SourceVary { + /// These request headers, in lowercase, sorted and each named once. + /// Empty when the response varies on nothing. + Headers(Vec), + /// `Vary: *`, which no other request matches. + Any, + /// `Cookie` is named, which keys a copy on every cookie a reader carries, + /// so each reader would have a copy of their own. + Cookie, + /// A `Vary` that is not a list of header names. + Unreadable, +} + +impl SourceVary { + /// What a response with `headers` says it varies on. + pub fn of(headers: &HeaderMap) -> Self { + let mut names = Vec::new(); + for value in headers.get_all(header::VARY) { + let Ok(value) = value.to_str() else { + return Self::Unreadable; + }; + for name in value + .split(',') + .map(str::trim) + .filter(|name| !name.is_empty()) + { + if name == "*" { + return Self::Any; + } + let Ok(name) = HeaderName::from_bytes(name.as_bytes()) else { + return Self::Unreadable; + }; + if name == header::COOKIE { + return Self::Cookie; + } + names.push(name.as_str().to_owned()); + } + } + names.sort_unstable(); + names.dedup(); + Self::Headers(names) + } + + /// The same, leaving out `name`, a header whose value never selects a + /// different copy, such as `Accept-Encoding` for a copy kept decoded and + /// encoded for each reader as it is served. + #[cfg(test)] + #[must_use] + pub fn without(self, name: &HeaderName) -> Self { + match self { + Self::Headers(mut names) => { + names.retain(|kept| kept != name.as_str()); + Self::Headers(names) + } + other => other, + } + } +} + +/// What the cache holds for an address. +#[derive(Debug, Clone, PartialEq, Eq)] +enum AddressRecord { + /// Each copy at the address is found by a request's values for these + /// headers. + VariesOn(Vec), + /// No copy at the address is served to another request. + NoCopies, +} + +impl AddressRecord { + fn encode(&self) -> Bytes { + match self { + Self::VariesOn(names) => Bytes::from(format!("v1:{}", names.join(","))), + Self::NoCopies => Bytes::from_static(b"v1*"), + } + } + + /// The record `bytes` hold, or `None` for bytes this build did not write. + fn decode(bytes: &[u8]) -> Option { + let text = std::str::from_utf8(bytes).ok()?; + if text == "v1*" { + return Some(Self::NoCopies); + } + let names = text.strip_prefix("v1:")?; + if names.is_empty() { + return Some(Self::VariesOn(Vec::new())); + } + names + .split(',') + .map(|name| { + HeaderName::from_bytes(name.as_bytes()) + .ok() + .filter(|parsed| parsed.as_str() == name) + .map(|_| name.to_owned()) + }) + .collect::>>() + .map(Self::VariesOn) + } +} + +/// One address, made of the publisher's prefix, so two publishers never share +/// a record or a copy, and a digest of what the request asks the source for. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceAddress { + prefix: ServerKeyPrefix, + key: String, +} + +impl SourceAddress { + /// The address `key` names for the publisher `prefix` names. The key + /// should be short, such as a digest, because a store refuses a long key. + #[must_use] + pub fn new(prefix: ServerKeyPrefix, key: String) -> Self { + Self { prefix, key } + } + + fn record_key(&self, cache: &SourceCache) -> StoreKey { + StoreKey::new(&self.prefix, cache.records.name, self.key.clone()) + } + + fn copy_key(&self, cache: &SourceCache, variant: &str) -> StoreKey { + StoreKey::new( + &self.prefix, + cache.copies.name, + format!("{}-{variant}", self.key), + ) + } +} + +/// The bytes of a variant's digest its key holds, half the digest. 128 bits +/// keeps two variants of one address from ever sharing a key, and a key is +/// read by stores that refuse long ones. +const VARIANT_DIGEST_BYTES: usize = 16; + +/// Which copy a request is served, as a digest of its values for `names` and +/// of what the publisher's rule declared for it. +/// +/// An absent header and one present with an empty value are different requests +/// to the source, so they give different digests, as do repeated values in a +/// different order. +fn variant(names: &[String], headers: &HeaderMap, declared: Option<&str>) -> String { + let mut hasher = Sha256::new(); + push(&mut hasher, b"ts-asset-variant-v1"); + push(&mut hasher, &(names.len() as u64).to_be_bytes()); + for name in names { + push(&mut hasher, name.as_bytes()); + let values: Vec<&[u8]> = headers + .get_all(name.as_str()) + .iter() + .map(http::HeaderValue::as_bytes) + .collect(); + if values.is_empty() { + push(&mut hasher, b"absent"); + } else { + push(&mut hasher, b"present"); + push(&mut hasher, &(values.len() as u64).to_be_bytes()); + for value in values { + push(&mut hasher, value); + } + } + } + match declared { + None => push(&mut hasher, b"undeclared"), + Some(token) => { + push(&mut hasher, b"declared"); + push(&mut hasher, token.as_bytes()); + } + } + hex::encode(&hasher.finalize()[..VARIANT_DIGEST_BYTES]) +} + +/// The two loading caches one kind of response uses, one holding each +/// address's record, and one holding the copies. +#[derive(Debug, Clone)] +pub struct SourceCache { + /// Holds each address's record of what its responses vary on. + pub records: CacheSpec, + /// Holds the copies, each under its address and a request's values. + pub copies: CacheSpec, +} + +/// What one fetch of the source gave the request that made it. +#[derive(Debug, Clone)] +pub struct Fetched { + /// What the source's response varies on. + pub vary: SourceVary, + /// The copy cache's value for the request that fetched it, in the caller's + /// own form, which may say the response may not be served to another + /// request. + pub copy: Bytes, + /// How long the copy may be used. The record lasts as long. + pub lifetime: Duration, +} + +/// The work of one [`Fetch`]. +pub type FetchFuture = Pin>>>; + +/// Fetches the source for one request, run at most once for its record and +/// once for its copy, by whichever of the two loads this request leads. +pub type Fetch = Arc FetchFuture + Send + Sync>; + +/// One request's lookup. +pub struct Lookup<'a> { + /// The request's services, whose loading cache holds the records and the + /// copies. + pub services: &'a RuntimeServices, + /// The caches this kind of response uses. + pub cache: &'a SourceCache, + /// What the request asks the source for. + pub address: &'a SourceAddress, + /// The request's headers, as they are sent to the source. + pub headers: &'a HeaderMap, + /// What the publisher's rule says separates this request's copy, beyond + /// the source's own `Vary`. + pub declared: Option<&'a str>, + /// The copy cache's value saying a response may not be served to another + /// request, in the caller's form. + pub not_shareable: &'a Bytes, +} + +/// Where a load leaves the copy its own fetch made, for the request whose +/// fetch it was. +type Own = Arc>>; + +fn keep_own(own: &Own, copy: &Bytes) { + *own.lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(copy.clone()); +} + +fn take_own(own: &Own) -> Option { + own.lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() +} + +/// What a lookup found. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Found { + /// The copy cache's value for this request's values, in the caller's form. + Copy { + /// The value. + value: Bytes, + /// Where it is kept, for a caller that finds it cannot use it. + key: StoreKey, + }, + /// No copy at the address is served to another request, so this request + /// fetches its own response. + NoCopies, +} + +/// The copy for `lookup`'s request, from the record and copy caches or from +/// `fetch`. When this request's own fetch ran, it is served what that fetch +/// made, even when the cache keeps it for nobody. +/// +/// # Errors +/// +/// The error of a fetch this request ran or waited on. Every request waiting +/// on that fetch receives it, and nothing is kept. +pub async fn get_or_fetch(lookup: &Lookup<'_>, fetch: Fetch) -> Result { + let own = Own::default(); + let record_key = lookup.address.record_key(lookup.cache); + let record = lookup + .services + .load_cached( + &lookup.cache.records, + &record_key, + record_loader(lookup, Arc::clone(&fetch), Arc::clone(&own)), + ) + .await?; + let Some(record) = AddressRecord::decode(&record) else { + log::warn!("source_cache: a record this build did not write was dropped"); + lookup + .services + .loading_cache_for(lookup.cache.records.name) + .remove(lookup.services, &lookup.cache.records, &record_key) + .await; + return Ok(Found::NoCopies); + }; + let names = match record { + AddressRecord::NoCopies => return Ok(Found::NoCopies), + AddressRecord::VariesOn(names) => names, + }; + let copy_key = lookup.address.copy_key( + lookup.cache, + &variant(&names, lookup.headers, lookup.declared), + ); + if let Some(value) = take_own(&own) { + return Ok(Found::Copy { + value, + key: copy_key, + }); + } + let value = lookup + .services + .load_cached( + &lookup.cache.copies, + ©_key, + copy_loader(lookup, names, fetch, Arc::clone(&own)), + ) + .await?; + Ok(Found::Copy { + value: take_own(&own).unwrap_or(value), + key: copy_key, + }) +} + +/// The record's load, which fetches the source and records what its response +/// varies on. The fetching request's copy is written under its own values +/// first, so every request waiting on the record finds it. +fn record_loader(lookup: &Lookup<'_>, fetch: Fetch, own: Own) -> Loader { + let services = lookup.services.clone().without_connection(); + let cache = lookup.cache.clone(); + let address = lookup.address.clone(); + let headers = lookup.headers.clone(); + let declared = lookup.declared.map(str::to_owned); + Box::new(move || { + Box::pin(async move { + let fetched = fetch().await?; + keep_own(&own, &fetched.copy); + let SourceVary::Headers(names) = fetched.vary else { + return Ok(LoadedValue::new(AddressRecord::NoCopies.encode()) + .with_lifetime(fetched.lifetime)); + }; + let copy_key = + address.copy_key(&cache, &variant(&names, &headers, declared.as_deref())); + let copy = fetched.copy; + let lifetime = fetched.lifetime; + services + .load_cached( + &cache.copies, + ©_key, + Box::new(move || { + Box::pin(async move { Ok(LoadedValue::new(copy).with_lifetime(lifetime)) }) + }), + ) + .await?; + Ok(LoadedValue::new(AddressRecord::VariesOn(names).encode()) + .with_lifetime(fetched.lifetime)) + }) + }) +} + +/// A copy's load, which fetches the source for this request's values. +/// +/// A response that no longer varies on what the record names cannot be kept +/// under a key the record made, so only the request that fetched it is served +/// it, and the record is dropped so the next request learns what the source +/// varies on now. +fn copy_loader(lookup: &Lookup<'_>, names: Vec, fetch: Fetch, own: Own) -> Loader { + let services = lookup.services.clone().without_connection(); + let cache = lookup.cache.clone(); + let address = lookup.address.clone(); + let not_shareable = lookup.not_shareable.clone(); + Box::new(move || { + Box::pin(async move { + let fetched = fetch().await?; + keep_own(&own, &fetched.copy); + if fetched.vary != SourceVary::Headers(names) { + log::debug!( + "source_cache: the source no longer varies as its record says, so the \ + record is dropped" + ); + services + .loading_cache_for(cache.records.name) + .remove(&services, &cache.records, &address.record_key(&cache)) + .await; + return Ok(LoadedValue::new(not_shareable).with_lifetime(Duration::ZERO)); + } + Ok(LoadedValue::new(fetched.copy).with_lifetime(fetched.lifetime)) + }) + }) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use http::HeaderValue; + + use super::*; + use trusted_server_core::platform::{ + CacheLifetimes, + test_support::{MemoryLoadingCache, StubHttpClient, build_services_with_http_client}, + }; + + fn services(cache: Arc) -> RuntimeServices { + build_services_with_http_client(Arc::new(StubHttpClient::new())).with_loading_cache(cache) + } + + fn spec(name: &'static str) -> CacheSpec { + CacheSpec { + name, + lifetimes: CacheLifetimes { + memory: Duration::from_secs(5), + store: Duration::from_secs(60), + store_idle: Duration::from_secs(60), + }, + memory_entries: 8, + } + } + + fn cache() -> SourceCache { + SourceCache { + records: spec("test-vary"), + copies: spec("test"), + } + } + + fn headers(pairs: &[(&'static str, &str)]) -> HeaderMap { + let mut headers = HeaderMap::new(); + for (name, value) in pairs { + headers.append( + HeaderName::from_static(name), + HeaderValue::from_str(value).expect("should be a header value"), + ); + } + headers + } + + /// A source answering with `vary`, whose copy names the edition the + /// request asked for, counting its fetches. + fn source(vary: SourceVary, edition: &str, fetches: &Arc) -> Fetch { + let fetches = Arc::clone(fetches); + let copy = Bytes::from(format!("edition {edition}")); + Arc::new(move || { + let fetches = Arc::clone(&fetches); + let vary = vary.clone(); + let copy = copy.clone(); + Box::pin(async move { + fetches.fetch_add(1, Ordering::SeqCst); + Ok(Fetched { + vary, + copy, + lifetime: Duration::from_secs(30), + }) + }) + }) + } + + fn copy(found: Found) -> Option { + match found { + Found::Copy { value, .. } => Some(value), + Found::NoCopies => None, + } + } + + async fn look_up( + services: &RuntimeServices, + request: &HeaderMap, + declared: Option<&str>, + fetch: Fetch, + ) -> Found { + let address = SourceAddress::new( + ServerKeyPrefix::new("ts", "example.com"), + "address".to_owned(), + ); + get_or_fetch( + &Lookup { + services, + cache: &cache(), + address: &address, + headers: request, + declared, + not_shareable: &Bytes::from_static(b"not shareable"), + }, + fetch, + ) + .await + .expect("the lookup should succeed") + } + + #[test] + fn record_and_copy_keys_follow_the_publisher() { + trusted_server_core::prefix::test_support::assert_keys_follow_the_publisher(|domain| { + let address = + SourceAddress::new(ServerKeyPrefix::new("ts", domain), "address".to_owned()); + format!( + "{} {}", + address.record_key(&cache()), + address.copy_key(&cache(), "variant") + ) + }); + } + + #[test] + fn vary_names_are_lowercase_sorted_and_named_once() { + let response = headers(&[ + ("vary", "X-Edition, Accept-Encoding"), + ("vary", "x-edition"), + ]); + + assert_eq!( + SourceVary::of(&response), + SourceVary::Headers(vec!["accept-encoding".to_owned(), "x-edition".to_owned()]) + ); + assert_eq!( + SourceVary::of(&response).without(&header::ACCEPT_ENCODING), + SourceVary::Headers(vec!["x-edition".to_owned()]), + "a header that never selects a copy is left out" + ); + assert_eq!( + SourceVary::of(&HeaderMap::new()), + SourceVary::Headers(Vec::new()) + ); + } + + #[test] + fn a_wildcard_a_cookie_or_an_unreadable_vary_allows_no_copies() { + for (vary, expected) in [ + ("*", SourceVary::Any), + ("x-edition, *", SourceVary::Any), + ("Cookie", SourceVary::Cookie), + ("x edition", SourceVary::Unreadable), + ] { + assert_eq!( + SourceVary::of(&headers(&[("vary", vary)])), + expected, + "{vary}" + ); + } + } + + #[test] + fn a_record_reads_back_as_written_and_a_foreign_one_does_not_read() { + for record in [ + AddressRecord::VariesOn(Vec::new()), + AddressRecord::VariesOn(vec!["accept-language".to_owned(), "x-edition".to_owned()]), + AddressRecord::NoCopies, + ] { + assert_eq!(AddressRecord::decode(&record.encode()), Some(record)); + } + for foreign in [&b"v2:x-edition"[..], b"", b"v1:X-Edition", b"v1:x edition"] { + assert_eq!(AddressRecord::decode(foreign), None, "{foreign:?}"); + } + } + + #[test] + fn a_variant_tells_absent_empty_and_reordered_values_apart() { + let names = vec!["x-edition".to_owned()]; + let digest = |pairs: &[(&'static str, &str)]| variant(&names, &headers(pairs), None); + + let absent = digest(&[]); + let empty = digest(&[("x-edition", "")]); + let one_then_two = digest(&[("x-edition", "one"), ("x-edition", "two")]); + let two_then_one = digest(&[("x-edition", "two"), ("x-edition", "one")]); + + assert_ne!(absent, empty); + assert_ne!(one_then_two, two_then_one); + assert_eq!( + digest(&[("x-edition", "one"), ("x-other", "a")]), + digest(&[("x-edition", "one"), ("x-other", "b")]), + "a header the source does not vary on does not separate copies" + ); + assert_ne!( + variant(&names, &HeaderMap::new(), Some("token")), + variant(&names, &HeaderMap::new(), None), + "what the publisher's rule declares separates copies" + ); + } + + #[tokio::test] + async fn the_first_response_records_its_vary_and_each_value_gets_its_own_copy() { + let memory = Arc::new(MemoryLoadingCache::default()); + let services = services(Arc::clone(&memory)); + let fetches = Arc::new(AtomicUsize::new(0)); + let vary = SourceVary::Headers(vec!["x-edition".to_owned()]); + let red = headers(&[("x-edition", "red")]); + let blue = headers(&[("x-edition", "blue")]); + + let first = look_up(&services, &red, None, source(vary.clone(), "red", &fetches)).await; + let again = look_up(&services, &red, None, source(vary.clone(), "red", &fetches)).await; + let other = look_up( + &services, + &blue, + None, + source(vary.clone(), "blue", &fetches), + ) + .await; + let other_again = look_up(&services, &blue, None, source(vary, "blue", &fetches)).await; + + assert_eq!(copy(first), Some(Bytes::from("edition red"))); + assert_eq!(copy(again), Some(Bytes::from("edition red"))); + assert_eq!(copy(other), Some(Bytes::from("edition blue"))); + assert_eq!(copy(other_again), Some(Bytes::from("edition blue"))); + assert_eq!( + fetches.load(Ordering::SeqCst), + 2, + "each value is fetched once, and the first fetch also made the record" + ); + } + + #[tokio::test] + async fn a_response_that_allows_no_copies_has_every_request_fetch_its_own() { + let memory = Arc::new(MemoryLoadingCache::default()); + let services = services(Arc::clone(&memory)); + let fetches = Arc::new(AtomicUsize::new(0)); + let request = headers(&[("x-edition", "red")]); + + for _ in 0..2 { + let found = look_up( + &services, + &request, + None, + source(SourceVary::Any, "red", &fetches), + ) + .await; + assert_eq!(found, Found::NoCopies); + } + assert_eq!( + fetches.load(Ordering::SeqCst), + 1, + "the record saying no copies are kept is itself kept" + ); + } + + #[tokio::test] + async fn a_source_that_changes_its_vary_has_its_record_dropped() { + let memory = Arc::new(MemoryLoadingCache::default()); + let services = services(Arc::clone(&memory)); + let fetches = Arc::new(AtomicUsize::new(0)); + let red = headers(&[("x-edition", "red")]); + let blue = headers(&[("x-edition", "blue")]); + look_up( + &services, + &red, + None, + source( + SourceVary::Headers(vec!["x-edition".to_owned()]), + "red", + &fetches, + ), + ) + .await; + + let changed = look_up( + &services, + &blue, + None, + source( + SourceVary::Headers(vec!["x-region".to_owned()]), + "blue", + &fetches, + ), + ) + .await; + + assert_eq!( + copy(changed), + Some(Bytes::from("edition blue")), + "the request that fetched a copy made under another Vary is served it" + ); + assert_eq!( + fetches.load(Ordering::SeqCst), + 2, + "that request fetches the source once" + ); + let keys = memory.keys(); + assert!( + keys.iter() + .filter_map(|key| memory.held(key)) + .all(|(value, _)| value != "edition blue"), + "no other request is served that copy" + ); + assert!( + keys.iter().all(|key| !key.contains("/test-vary/")), + "the record is dropped so the next request learns the new Vary: {keys:?}" + ); + } + + #[tokio::test] + async fn the_publishers_rule_separates_copies_the_source_does_not() { + let memory = Arc::new(MemoryLoadingCache::default()); + let services = services(Arc::clone(&memory)); + let fetches = Arc::new(AtomicUsize::new(0)); + let request = HeaderMap::new(); + let nothing = SourceVary::Headers(Vec::new()); + + let uk = look_up( + &services, + &request, + Some("uk"), + source(nothing.clone(), "uk", &fetches), + ) + .await; + let us = look_up( + &services, + &request, + Some("us"), + source(nothing.clone(), "us", &fetches), + ) + .await; + let uk_again = look_up( + &services, + &request, + Some("uk"), + source(nothing, "x", &fetches), + ) + .await; + + assert_eq!(copy(uk), Some(Bytes::from("edition uk"))); + assert_eq!(copy(us), Some(Bytes::from("edition us"))); + assert_eq!(copy(uk_again), Some(Bytes::from("edition uk"))); + assert_eq!(fetches.load(Ordering::SeqCst), 2); + } +} diff --git a/crates/asset-cache/51degrees/src/tests.rs b/crates/asset-cache/51degrees/src/tests.rs new file mode 100644 index 0000000000..ab6162e934 --- /dev/null +++ b/crates/asset-cache/51degrees/src/tests.rs @@ -0,0 +1,1055 @@ +//! The module's tests, which drive it through `AssetCacheModule::answer` as +//! core does, with `StubHttpClient` as the origin and `MemoryLoadingCache` as +//! the loading cache. + +use std::sync::Arc; +use std::time::Duration; + +use edgezero_core::body::Body as EdgeBody; +use http::{HeaderMap, HeaderName, HeaderValue, Response, StatusCode, Uri, header}; +use trusted_server_core::asset_cache::{ + AssetAnswer, AssetCacheModule, AssetFile, AssetVia, OriginRequest, +}; +use trusted_server_core::integrations::{IntegrationBuilder, IntegrationRegistry}; +use trusted_server_core::platform::test_support::{ + MemoryLoadingCache, StubHttpClient, build_services_with_http_client, +}; +use trusted_server_core::platform::{LoadingCaches, PlatformLoadingCache, RuntimeServices}; +use trusted_server_core::settings::{LoadingCacheConfig, Settings}; + +use crate::entry::{AssetEntry, MAX_ASSET_COPY_BYTES}; +use crate::{ASSETS_CACHE, ASSETS_VARY_CACHE, AssetCache, MODULE, builder}; + +/// The cache policy of a file the origin lets be kept for five minutes. +const KEPT_FOR_FIVE_MINUTES: &str = "public, max-age=300"; + +/// The most bytes a test reads of a response body. +const MAX_BODY_BYTES: usize = 4 * 1024 * 1024; + +/// The host the reader asked. +const READER_HOST: &str = "www.example.com"; + +/// A stylesheet the publisher's origin sends. +const STYLESHEET_URL: &str = "https://origin.example.com/site.css"; + +/// A stylesheet an asset route fetches from its origin. +const ROUTE_STYLESHEET_URL: &str = "https://assets.example.com/.images/site.css"; + +/// What names the route's origin, being its origin URL and prefix. +const ROUTE_IDENTITY: &str = "https://assets.example.com\0/.images/"; + +const ORIGIN_BACKEND: &str = "origin-backend"; + +fn settings() -> Settings { + trusted_server_core::test_support::tests::create_test_settings() +} + +fn module() -> AssetCache { + AssetCache::from_settings(&settings()) +} + +fn services(stub: &Arc, cache: &Arc) -> RuntimeServices { + build_services_with_http_client(Arc::clone(stub) as _) + .with_loading_cache(Arc::clone(cache) as _) +} + +fn headers(pairs: &[(&'static str, &str)]) -> HeaderMap { + let mut headers = HeaderMap::new(); + for (name, value) in pairs { + headers.append( + HeaderName::from_static(name), + HeaderValue::from_str(value).expect("should be a header value"), + ); + } + headers +} + +/// The request core would send for `url`, carrying `request_headers`. +fn origin(url: &str, request_headers: HeaderMap, streaming: bool) -> OriginRequest { + let uri = url.parse::().expect("should parse the URL"); + OriginRequest::for_test(uri, request_headers, ORIGIN_BACKEND, streaming) +} + +/// A file the publisher path offers for `url`, carrying `request_headers`. +fn publisher_file(url: &'static str, request_headers: HeaderMap) -> AssetFile<'static> { + AssetFile::for_test( + AssetVia::Publisher, + READER_HOST, + url, + origin(url, request_headers, false), + ) +} + +/// A file the asset route offers. +fn route_file() -> AssetFile<'static> { + let mut file = AssetFile::for_test( + AssetVia::Route, + READER_HOST, + ROUTE_STYLESHEET_URL, + origin(ROUTE_STYLESHEET_URL, HeaderMap::new(), false), + ); + file.origin_identity = ROUTE_IDENTITY; + file +} + +/// Queues the origin's answer, a `200` of `content_type` with `extra`. +fn queue(stub: &StubHttpClient, content_type: &str, body: &[u8], extra: &[(&str, &str)]) { + let mut response_headers = vec![("content-type", content_type)]; + response_headers.extend_from_slice(extra); + stub.push_response_with_headers(200, body.to_vec(), response_headers); +} + +/// What the reader is served for `file`, as core serves it: the module's +/// answer, or the origin's response when the module passes the file back. +async fn serve( + module: &AssetCache, + services: &RuntimeServices, + file: AssetFile<'_>, +) -> Result, String> { + let origin = file.origin.clone(); + match module.answer(services, file).await { + AssetAnswer::Serve(response) => Ok(response), + AssetAnswer::OriginFailed(report) => Err(format!("{report:?}")), + AssetAnswer::AskOrigin => services + .http_client() + .send(origin.platform_request()) + .await + .map(|sent| sent.response) + .map_err(|report| format!("{report:?}")), + } +} + +async fn served_body( + module: &AssetCache, + services: &RuntimeServices, + file: AssetFile<'_>, +) -> Vec { + let response = serve(module, services, file) + .await + .expect("should serve the file"); + body_of(response).await +} + +async fn body_of(response: Response) -> Vec { + response + .into_body() + .into_bytes_bounded(MAX_BODY_BYTES) + .await + .expect("should read the body") + .to_vec() +} + +/// The time left to each copy `cache` holds, leaving out the records and the +/// marks of what may not be shared, in no order. +fn asset_copy_lifetimes(cache: &MemoryLoadingCache) -> Vec { + let copies = format!("/{ASSETS_CACHE}/"); + cache + .keys() + .iter() + .filter(|key| key.contains(&copies)) + .filter_map(|key| cache.held(key)) + .filter(|(value, _)| matches!(AssetEntry::decode(value), Ok(AssetEntry::Copy(_)))) + .map(|(_, left)| left.unwrap_or(Duration::MAX)) + .collect() +} + +#[tokio::test] +async fn an_image_is_kept_as_sent_and_one_fetch_serves_three_requests() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + let image = [0x89_u8, b'P', b'N', b'G', 0, 1, 2, 3]; + queue( + &stub, + "image/png", + &image, + &[("cache-control", KEPT_FOR_FIVE_MINUTES)], + ); + + for _ in 0..3 { + let file = publisher_file("https://origin.example.com/a.png", HeaderMap::new()); + assert_eq!(served_body(&module, &services, file).await, image); + } + + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "one fetch serves all three" + ); +} + +#[tokio::test] +async fn a_stylesheet_is_served_from_one_fetch_with_its_age() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + queue( + &stub, + "text/css", + b"body{color:red}", + &[("cache-control", KEPT_FOR_FIVE_MINUTES)], + ); + + let first = served_body( + &module, + &services, + publisher_file(STYLESHEET_URL, HeaderMap::new()), + ) + .await; + let second = serve( + &module, + &services, + publisher_file(STYLESHEET_URL, HeaderMap::new()), + ) + .await + .expect("should serve the copy"); + assert!( + second.headers().contains_key(header::AGE), + "a kept copy says how long it has been kept" + ); + let second = body_of(second).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "one fetch serves both" + ); + assert_eq!( + second, first, + "the copy is what the first reader was served" + ); +} + +#[tokio::test] +async fn a_wildcard_vary_a_cookie_vary_a_private_or_a_no_store_response_is_never_kept() { + for (cache_control, vary, kept) in [ + (KEPT_FOR_FIVE_MINUTES, None, true), + (KEPT_FOR_FIVE_MINUTES, Some("*"), false), + (KEPT_FOR_FIVE_MINUTES, Some("Cookie"), false), + ("private, max-age=300", None, false), + ("no-store", None, false), + ("public, max-age=300, no-store", None, false), + ] { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + for _ in 0..2 { + let mut extra = vec![("cache-control", cache_control)]; + extra.extend(vary.map(|vary| ("vary", vary))); + queue(&stub, "text/css", b"body{}", &extra); + let file = publisher_file(STYLESHEET_URL, HeaderMap::new()); + assert_eq!(served_body(&module, &services, file).await, b"body{}"); + } + + assert_eq!( + stub.recorded_request_uris().len(), + if kept { 1 } else { 2 }, + "Cache-Control {cache_control} and Vary {vary:?}" + ); + assert_eq!( + asset_copy_lifetimes(&cache).is_empty(), + !kept, + "Cache-Control {cache_control} and Vary {vary:?}" + ); + } +} + +#[tokio::test] +async fn s_maxage_then_max_age_set_how_long_a_copy_is_kept() { + for (cache_control, seconds) in [ + ("public, max-age=300", 300), + ("public, max-age=300, s-maxage=60", 60), + ("public, s-maxage=600, max-age=30", 600), + ] { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + queue( + &stub, + "text/css", + b"body{}", + &[("cache-control", cache_control)], + ); + + let file = publisher_file(STYLESHEET_URL, HeaderMap::new()); + let _ = served_body(&module(), &services, file).await; + + let lifetimes = asset_copy_lifetimes(&cache); + assert_eq!(lifetimes.len(), 1, "{cache_control}"); + let expected = Duration::from_secs(seconds); + assert!( + lifetimes[0] <= expected && lifetimes[0] > expected - Duration::from_secs(5), + "{cache_control} should keep a copy for {expected:?}, not {:?}", + lifetimes[0] + ); + } +} + +#[tokio::test] +async fn a_header_the_origin_varies_on_separates_kept_copies() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + for edition in ["red", "blue"] { + queue( + &stub, + "text/css", + format!("/* {edition} */").as_bytes(), + &[ + ("cache-control", KEPT_FOR_FIVE_MINUTES), + ("vary", "X-Edition"), + ], + ); + } + + for edition in ["red", "blue", "red", "blue"] { + let file = publisher_file(STYLESHEET_URL, headers(&[("x-edition", edition)])); + let css = served_body(&module, &services, file).await; + assert_eq!(css, format!("/* {edition} */").into_bytes()); + } + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "each edition is fetched once" + ); +} + +#[tokio::test] +async fn a_body_that_outgrows_a_copy_is_served_whole_from_the_one_fetch() { + for streaming in [false, true] { + let stub = Arc::new(StubHttpClient::new()); + stub.set_streaming_responses_supported(streaming); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let large = vec![b'a'; MAX_ASSET_COPY_BYTES + 1]; + for _ in 0..2 { + queue( + &stub, + "text/plain", + &large, + &[("cache-control", KEPT_FOR_FIVE_MINUTES)], + ); + } + let url = "https://origin.example.com/big.txt"; + let file = AssetFile::for_test( + AssetVia::Publisher, + READER_HOST, + url, + origin(url, HeaderMap::new(), streaming), + ); + + let body = served_body(&module(), &services, file).await; + + assert_eq!( + body, large, + "streaming {streaming}: the reader is served the whole body" + ); + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "streaming {streaming}: a body that outgrows a copy is served from the one fetch" + ); + assert!(asset_copy_lifetimes(&cache).is_empty()); + } +} + +#[tokio::test] +async fn a_response_whose_vary_changed_is_served_from_its_one_fetch() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + for (edition, vary) in [ + ("red", "X-Edition"), + ("blue", "X-Region"), + ("blue", "X-Region"), + ] { + queue( + &stub, + "text/css", + format!("/* {edition} */").as_bytes(), + &[("cache-control", KEPT_FOR_FIVE_MINUTES), ("vary", vary)], + ); + } + + let red = publisher_file(STYLESHEET_URL, headers(&[("x-edition", "red")])); + let blue = publisher_file(STYLESHEET_URL, headers(&[("x-edition", "blue")])); + let red = served_body(&module, &services, red).await; + let blue = served_body(&module, &services, blue).await; + + assert_eq!(red, b"/* red */"); + assert_eq!( + blue, b"/* blue */", + "the request whose fetch found the new Vary is served that fetch" + ); + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "each request fetches the origin once" + ); +} + +#[tokio::test] +async fn a_load_that_fails_is_the_answer_and_the_next_request_loads_again() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + + // Nothing is queued, so the stub fails the origin request. + let answer = module + .answer(&services, publisher_file(STYLESHEET_URL, HeaderMap::new())) + .await; + assert!( + matches!(answer, AssetAnswer::OriginFailed(_)), + "a failed load is the answer: {answer:?}" + ); + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the origin is asked once" + ); + assert!(cache.keys().is_empty(), "nothing is kept for a failed load"); + + queue( + &stub, + "text/css", + b"body{}", + &[("cache-control", KEPT_FOR_FIVE_MINUTES)], + ); + let file = publisher_file(STYLESHEET_URL, HeaderMap::new()); + assert_eq!( + served_body(&module, &services, file).await, + b"body{}", + "the next request loads the file again" + ); +} + +#[tokio::test] +async fn a_request_carrying_authorization_or_a_cookie_is_fetched_for_itself() { + type Mark = fn(&mut AssetFile<'static>); + let marks: [(&str, Mark); 2] = [ + ("Authorization", |file| file.carries_authorization = true), + ("Cookie", |file| file.carries_cookie = true), + ]; + for (carried, mark) in marks { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + for reader in ["one", "two"] { + queue( + &stub, + "application/json", + format!("{{\"reader\":\"{reader}\"}}").as_bytes(), + &[("cache-control", "max-age=300")], + ); + let mut file = publisher_file(STYLESHEET_URL, HeaderMap::new()); + mark(&mut file); + let origin = file.origin.clone(); + + let answer = module.answer(&services, file).await; + assert!( + matches!(answer, AssetAnswer::AskOrigin), + "a request carrying {carried} is passed back to core: {answer:?}" + ); + let sent = services + .http_client() + .send(origin.platform_request()) + .await + .expect("core asks the origin"); + assert_eq!( + body_of(sent.response).await, + format!("{{\"reader\":\"{reader}\"}}").into_bytes(), + "reader {reader} is served the answer made for them" + ); + } + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "each request carrying {carried} is fetched for itself" + ); + assert!( + asset_copy_lifetimes(&cache).is_empty(), + "no answer to a request carrying {carried} is kept" + ); + } +} + +#[tokio::test] +async fn an_answer_that_sets_a_cookie_is_served_only_to_the_reader_it_was_made_for() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + for reader in ["one", "two"] { + let cookie = format!("visitor={reader}"); + queue( + &stub, + "text/css", + format!("/* {reader} */").as_bytes(), + &[ + ("cache-control", KEPT_FOR_FIVE_MINUTES), + ("set-cookie", cookie.as_str()), + ], + ); + } + + for reader in ["one", "two"] { + let file = publisher_file(STYLESHEET_URL, HeaderMap::new()); + let response = serve(&module, &services, file) + .await + .expect("should serve the file"); + let cookie = format!("visitor={reader}"); + assert!( + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .any(|value| value.as_bytes() == cookie.as_bytes()), + "reader {reader} is served the cookie the origin set for them" + ); + assert_eq!( + body_of(response).await, + format!("/* {reader} */").into_bytes() + ); + } + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "each reader's request is fetched for itself" + ); + assert!( + asset_copy_lifetimes(&cache).is_empty(), + "an answer that sets a cookie is kept for no one" + ); +} + +/// A loading cache module's cache, which keeps what it loads. +fn memory_module_cache() -> Arc { + Arc::new(MemoryLoadingCache::default()) +} + +/// A loading cache module, and this module, as a deployment links them. +fn linked_builders() -> [IntegrationBuilder; 2] { + [ + IntegrationBuilder::implementations("loading-cache.memory", "a-crate") + .with_loading_cache(memory_module_cache), + builder(), + ] +} + +#[tokio::test] +async fn one_fetch_serves_two_requests_when_the_settings_name_no_loading_cache_module() { + let caches = + trusted_server_core::integrations::loading_caches_from(&settings(), &linked_builders()) + .expect("should select the module's caches"); + let stub = Arc::new(StubHttpClient::new()); + let services = + build_services_with_http_client(Arc::clone(&stub) as _).with_loading_caches(caches); + let module = module(); + queue( + &stub, + "text/css", + b"body{}", + &[("cache-control", KEPT_FOR_FIVE_MINUTES)], + ); + + for _ in 0..2 { + let file = publisher_file(STYLESHEET_URL, HeaderMap::new()); + assert_eq!(served_body(&module, &services, file).await, b"body{}"); + } + + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "both caches take the one loading cache module the build links" + ); +} + +#[tokio::test] +async fn a_document_is_never_answered_from_a_copy() { + let stub = Arc::new(StubHttpClient::new()); + queue( + &stub, + "text/css", + b"body{}", + &[("cache-control", KEPT_FOR_FIVE_MINUTES)], + ); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let mut file = publisher_file(STYLESHEET_URL, HeaderMap::new()); + file.is_document = true; + + let answer = module().answer(&services, file).await; + + assert!( + matches!(answer, AssetAnswer::AskOrigin), + "a document is passed back, however shareable its origin's answer: {answer:?}" + ); + assert!( + stub.recorded_request_uris().is_empty(), + "the module asks no origin for a document" + ); +} + +fn whole_body_stub() -> Arc { + let stub = Arc::new(StubHttpClient::new()); + stub.set_streaming_responses_supported(false); + stub +} + +fn queue_route_stylesheet(stub: &StubHttpClient) { + queue( + stub, + "text/css", + b"body { color: red }", + &[("cache-control", KEPT_FOR_FIVE_MINUTES)], + ); +} + +#[tokio::test] +async fn a_routes_answer_is_held_and_serves_the_next_request() { + let stub = whole_body_stub(); + queue_route_stylesheet(&stub); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + + let first = serve(&module, &services, route_file()) + .await + .expect("should serve the file"); + let second = serve(&module, &services, route_file()) + .await + .expect("should serve the copy"); + + assert_eq!(first.status(), StatusCode::OK); + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the origin is asked once for two requests of one file" + ); + assert_eq!(second.status(), StatusCode::OK); + assert_eq!( + second + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("text/css"), + "the copy is served with the headers that describe it" + ); + assert!( + second.headers().contains_key(header::AGE), + "the copy is served with an Age" + ); + assert_eq!(body_of(second).await, b"body { color: red }"); + let lifetimes = asset_copy_lifetimes(&cache); + assert!( + lifetimes.len() == 1 + && lifetimes[0] > Duration::from_secs(295) + && lifetimes[0] <= Duration::from_secs(300), + "the copy is kept for the lifetime the origin gave it, less the time since: \ + {lifetimes:?}" + ); +} + +#[tokio::test] +async fn a_routes_answer_that_sets_a_cookie_is_fetched_each_time() { + let stub = whole_body_stub(); + for _ in 0..2 { + queue( + &stub, + "text/css", + b"body { color: red }", + &[ + ("cache-control", KEPT_FOR_FIVE_MINUTES), + ("set-cookie", "session=1"), + ], + ); + } + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + + for _ in 0..2 { + let response = serve(&module, &services, route_file()) + .await + .expect("should serve the file"); + assert_eq!(response.status(), StatusCode::OK); + } + + assert_eq!(stub.recorded_request_uris().len(), 2); + assert!(asset_copy_lifetimes(&cache).is_empty(), "no copy is kept"); +} + +#[tokio::test] +async fn a_readers_cookie_does_not_keep_a_routes_answer_from_a_copy() { + let stub = whole_body_stub(); + queue_route_stylesheet(&stub); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + + for _ in 0..2 { + let mut file = route_file(); + file.carries_cookie = true; + let response = serve(&module, &services, file) + .await + .expect("should serve the file"); + assert_eq!(response.status(), StatusCode::OK); + } + + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "a route sends its origin no cookie, so both readers share one fetch" + ); +} + +#[tokio::test] +async fn a_routes_file_that_asks_for_the_origin_is_fetched() { + let stub = whole_body_stub(); + queue_route_stylesheet(&stub); + queue_route_stylesheet(&stub); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + let module = module(); + + let _ = served_body(&module, &services, route_file()).await; + let mut asking = route_file(); + asking.asks_for_origin = true; + let answer = module.answer(&services, asking).await; + + assert!( + matches!(answer, AssetAnswer::AskOrigin), + "a range, a conditional header, or a Cache-Control or Pragma that asks \ + for the origin is passed back to core: {answer:?}" + ); + assert_eq!(stub.recorded_request_uris().len(), 1); +} + +#[tokio::test] +async fn nothing_is_held_while_the_caches_keep_no_values() { + let stub = whole_body_stub(); + queue_route_stylesheet(&stub); + queue_route_stylesheet(&stub); + let services = build_services_with_http_client(Arc::clone(&stub) as _); + let module = module(); + + for _ in 0..2 { + let answer = module.answer(&services, route_file()).await; + assert!( + matches!(answer, AssetAnswer::AskOrigin), + "a module whose caches keep nothing passes every file back: {answer:?}" + ); + } + + assert!( + stub.recorded_request_uris().is_empty(), + "the module asks no origin itself" + ); +} + +#[tokio::test] +async fn a_routes_load_that_fails_is_the_answer_after_one_origin_request() { + let stub = whole_body_stub(); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services(&stub, &cache); + + // Nothing is queued, so the stub fails the origin request. + let answer = module().answer(&services, route_file()).await; + + assert!( + matches!(answer, AssetAnswer::OriginFailed(_)), + "a failed load is the answer: {answer:?}" + ); + assert_eq!(stub.recorded_request_uris().len(), 1); +} + +#[test] +fn the_builder_declares_the_two_caches() { + let builder = builder(); + + assert_eq!(builder.module_name(), Some(MODULE)); + assert_eq!(builder.cache_features(), [ASSETS_CACHE, ASSETS_VARY_CACHE]); +} + +#[test] +fn the_assets_table_sizes_both_caches_and_the_defaults_fill_what_it_leaves_unset() { + let mut settings = settings(); + settings.cache.declared.insert( + ASSETS_CACHE.to_owned(), + LoadingCacheConfig { + store_seconds: Some(600), + ..LoadingCacheConfig::default() + }, + ); + + let module = AssetCache::from_settings(&settings); + + for spec in [&module.caches.records, &module.caches.copies] { + assert_eq!( + spec.lifetimes.store, + Duration::from_secs(600), + "{}", + spec.name + ); + assert_eq!( + spec.lifetimes.memory, + Duration::from_secs(5), + "{}", + spec.name + ); + assert_eq!( + spec.lifetimes.store_idle, + Duration::from_secs(24 * 60 * 60), + "{}", + spec.name + ); + } + assert_eq!(module.caches.records.name, ASSETS_VARY_CACHE); + assert_eq!(module.caches.copies.name, ASSETS_CACHE); +} + +/// Guards core's check of the lifetimes in a declared cache's table, for the +/// table this module's builder declares. +#[test] +fn core_refuses_a_lifetime_out_of_range_in_the_assets_table() { + let toml = format!( + "{}\n[cache.51degrees-assets]\nstore_seconds = 0\n", + trusted_server_core::test_support::tests::crate_test_settings_str() + ); + + let error = Settings::from_toml(&toml).expect_err("a lifetime out of range should not load"); + + let message = format!("{error:?}"); + assert!( + message.contains( + "[cache.51degrees-assets] store_seconds is 0, and must be from 1 to 31536000" + ), + "{message}" + ); +} + +/// The test settings with `tables` after them, loaded from a document. +fn settings_with_tables(tables: &str) -> Settings { + let toml = format!( + "{}\n{tables}", + trusted_server_core::test_support::tests::crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should load the document") +} + +/// The records take the lifetimes of `[cache.51degrees-assets]`, so a +/// lifetime in `[cache.51degrees-assets-vary]` is refused when the settings +/// load, whether or not `[asset-cache]` selects the module. +#[test] +fn each_lifetime_in_the_vary_table_is_refused_naming_the_table_and_the_setting() { + // (setting, a value in its range) + let lifetimes = [ + ("memory_seconds", "5"), + ("store_seconds", "3600"), + ("store_idle_seconds", "86400"), + ]; + for (setting, value) in lifetimes { + for selected in [false, true] { + let mut settings = settings_with_tables(&format!( + "[cache.51degrees-assets-vary]\n{setting} = {value}\n" + )); + if selected { + settings.select_module("asset-cache", MODULE); + } + + let mut refusals = vec![ + ( + "loading the settings", + trusted_server_core::config::validate_settings_for_runtime_with( + &settings, + &[builder()], + ) + .err(), + ), + ( + "the deploy check", + trusted_server_core::config::validate_settings_for_deploy_with( + &settings, + &[builder()], + ) + .err(), + ), + ]; + // The registry builds only a module a section selects. + if selected { + refusals.push(( + "the registry", + IntegrationRegistry::with_registrations(&settings, &[builder()]).err(), + )); + } + + for (call, refusal) in refusals { + let message = format!( + "{:?}", + refusal.unwrap_or_else(|| panic!( + "{call} should refuse `{setting}`, selected {selected}" + )) + ); + assert!( + message.contains(&format!( + "[cache.51degrees-assets-vary] sets `{setting}`, which" + )) && message.contains("[cache.51degrees-assets]."), + "{call} should name the table, the setting and where lifetimes go, \ + selected {selected}: {message}" + ); + for (other, _) in lifetimes.iter().filter(|(other, _)| *other != setting) { + assert!( + !message.contains(&format!("`{other}`")), + "{call} should name only `{setting}`: {message}" + ); + } + } + } + } +} + +#[test] +fn the_vary_table_refusal_names_every_lifetime_it_sets() { + let settings = settings_with_tables( + "[cache.51degrees-assets-vary]\nmemory_seconds = 5\nstore_seconds = 3600\n\ + store_idle_seconds = 86400\n", + ); + + let error = crate::validate(&settings).expect_err("no lifetime of the table is read"); + + let message = format!("{error:?}"); + assert!( + message.contains( + "[cache.51degrees-assets-vary] sets `memory_seconds`, `store_seconds` and \ + `store_idle_seconds`, which" + ), + "{message}" + ); +} + +/// A vary table that sets no lifetime loads, empty or naming its loading +/// cache module, beside an assets table that sets all three. +#[test] +fn a_vary_table_without_lifetimes_loads() { + for vary in ["", "module = \"memory\"\n"] { + let mut settings = settings_with_tables(&format!( + "[cache.51degrees-assets]\nmemory_seconds = 5\nstore_seconds = 3600\n\ + store_idle_seconds = 86400\n\n[cache.51degrees-assets-vary]\n{vary}" + )); + settings.select_module("asset-cache", MODULE); + let builders = linked_builders(); + + assert!( + settings.cache.declared.contains_key(ASSETS_VARY_CACHE), + "the document should hold the vary table: {vary:?}" + ); + trusted_server_core::config::validate_settings_for_runtime_with(&settings, &builders) + .expect("loading the settings should accept the table"); + trusted_server_core::config::validate_settings_for_deploy_with(&settings, &builders) + .expect("the deploy check should accept the table"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("the registry should accept the table"); + assert!(registry.asset_cache_module().is_some(), "{vary:?}"); + trusted_server_core::integrations::loading_caches_from(&settings, &builders) + .expect("the caches should be made"); + } +} + +#[tokio::test] +async fn a_record_cache_that_keeps_nothing_beside_one_that_does_has_the_origin_asked() { + let stub = Arc::new(StubHttpClient::new()); + let mut caches = LoadingCaches::default(); + caches.insert(ASSETS_CACHE, memory_module_cache()); + let services = build_services_with_http_client(Arc::clone(&stub) as _) + .with_loading_caches(Arc::new(caches)); + + let answer = module() + .answer(&services, publisher_file(STYLESHEET_URL, HeaderMap::new())) + .await; + + assert!( + matches!(answer, AssetAnswer::AskOrigin), + "every request would run the record's load, so none is saved: {answer:?}" + ); + assert!(stub.recorded_request_uris().is_empty()); +} + +#[test] +fn an_assets_table_naming_none_is_refused() { + let mut settings = settings(); + settings.cache.declared.insert( + ASSETS_CACHE.to_owned(), + LoadingCacheConfig { + module: Some("none".to_owned()), + ..LoadingCacheConfig::default() + }, + ); + + let error = + trusted_server_core::integrations::loading_caches_from(&settings, &linked_builders()) + .expect_err("no linked crate offers a loading cache named `none`"); + + let message = format!("{error:?}"); + assert!( + message.contains("[cache.51degrees-assets] module names `none`"), + "{message}" + ); +} + +#[test] +fn a_table_of_the_modules_own_is_refused_even_when_empty() { + for table in [serde_json::json!({}), serde_json::json!({ "typo": true })] { + let mut settings = settings(); + settings + .insert_module_config("asset-cache", MODULE, &table) + .expect("should write the table"); + + let checked = crate::validate(&settings).expect_err("the module reads no table"); + let built = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .err() + .expect("the module reads no table"); + + for message in [format!("{checked:?}"), format!("{built:?}")] { + assert!( + message.contains("[asset-cache.51degrees] is a table the module does not read"), + "{table}: {message}" + ); + } + } +} + +#[test] +fn asset_cache_selects_the_module() { + let mut selected = settings(); + selected.select_module("asset-cache", MODULE); + + let registry = IntegrationRegistry::with_registrations(&selected, &[builder()]) + .expect("should build the registry"); + let unselected = IntegrationRegistry::with_registrations(&settings(), &[builder()]) + .expect("should build the registry"); + + assert!(registry.asset_cache_module().is_some()); + assert!( + unselected.asset_cache_module().is_none(), + "a linked module holds nothing until [asset-cache] selects it" + ); +} + +#[test] +fn module_constant_is_the_crate_folder() { + assert_eq!( + MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); +} diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index a431044d37..7f51085867 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -275,8 +275,9 @@ fn refuse_platform_geo(settings: &Settings) -> Result<(), Report RuntimeServices { let client_info = ctx .request() @@ -528,6 +529,9 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime if let Some(module) = state.registry.device_module() { services = services.with_device_module(module); } + if let Some(module) = state.registry.asset_cache_module() { + services = services.with_asset_cache_module(module); + } // The publisher this request resolved to, its tenant included, names // every key in a shared store and every name the browser holds. let prefixes = state.registry.prefixes( diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 20a4bdf121..2a16036275 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -327,8 +327,9 @@ impl AppState { /// Build per-request [`RuntimeServices`], carrying the Edge Cookie module /// the composition root already resolved and applying the module-supplied /// geo, Edge Cookie and device modules selected by `[geo]`, `[ec]` and - /// `[device] module`. The services also carry the permission policy the - /// domain's settings loaded, whichever services the request is given. + /// `[device] module`, and the asset cache module `[asset-cache]` selects. + /// The services also carry the permission policy the domain's settings + /// loaded, whichever services the request is given. /// /// No Edge Cookie module is carried when the composition root found /// nothing safe to keep, so the request path resolves the selection for @@ -363,6 +364,9 @@ impl AppState { if let Some(module) = self.registry.device_module() { services = services.with_device_module(module); } + if let Some(module) = self.registry.asset_cache_module() { + services = services.with_asset_cache_module(module); + } // The publisher this request resolved to names every key in a shared // store and every name the browser holds. let prefixes = self.registry.prefixes( diff --git a/crates/trusted-server-core/src/asset_cache.rs b/crates/trusted-server-core/src/asset_cache.rs new file mode 100644 index 0000000000..25a0ac804b --- /dev/null +++ b/crates/trusted-server-core/src/asset_cache.rs @@ -0,0 +1,293 @@ +//! Where a module may answer for a file before core asks the file's origin. +//! +//! The publisher path and the asset routes each offer a `GET` for a file to +//! the asset cache module that `[asset-cache]` selects, at the one point +//! where core is about to ask the file's origin. The module answers with a +//! response, with the failure of the origin request it made, or by passing +//! the file back to core. Core keeps no copy, no key and no gate of its own, +//! so a deployment that selects no asset cache module has every file asked +//! of its origin. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::{HeaderMap, Method, Request, Response, Uri, header}; + +use crate::platform::{PlatformCacheIntent, PlatformError, PlatformHttpRequest, RuntimeServices}; + +/// The type folder, and the section, that selects the asset cache module. +pub const MODULE_TYPE: &str = "asset-cache"; + +/// A module that may answer for a file before core asks the file's origin. +#[async_trait::async_trait(?Send)] +pub trait AssetCacheModule: Send + Sync { + /// The answer for `file`, at the point core is about to ask its origin. + async fn answer(&self, services: &RuntimeServices, file: AssetFile<'_>) -> AssetAnswer; +} + +/// What core does with the module's answer. +#[derive(Debug)] +pub enum AssetAnswer { + /// Serves this in place of the origin's response, processed as that + /// response would be. Where the adapter does not stream responses, the + /// body must be whole, as the origin's would be. + Serve(Response), + /// The module asked the origin through [`AssetFile::origin`] and could + /// not reach it, so core answers as it does when its own origin request + /// fails. + OriginFailed(Report), + /// Core asks the origin itself, as it does with no module selected. + AskOrigin, +} + +/// Which of core's two paths offers a file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AssetVia { + /// [`handle_publisher_request`](crate::publisher::handle_publisher_request), + /// for a file the publisher's origin sends. + Publisher, + /// [`handle_asset_proxy_request`](crate::proxy::handle_asset_proxy_request), + /// for a file an asset route fetches. + Route, +} + +/// A `GET` core is about to send to a file's origin, with what core knows of +/// the reader's request. Core makes it, and a module reads its fields. +#[derive(Debug)] +#[non_exhaustive] +pub struct AssetFile<'a> { + /// The path that offers the file. + pub via: AssetVia, + /// Whether the reader's request carries `Authorization`. + pub carries_authorization: bool, + /// Whether the reader's request carries `Cookie`. + pub carries_cookie: bool, + /// The host the reader asked, empty when the request names none. + pub host: &'a str, + /// The scheme the reader asked over. + pub scheme: &'a str, + /// Whether the reader's request is for a document, such as a page or a + /// frame. + pub is_document: bool, + /// Whether the request, as core will send it, asks for the origin's own + /// answer, by a range, a conditional header, or a `Cache-Control` or + /// `Pragma` that asks for the origin. On the publisher path also when a + /// module left state on the request or an integration marked its + /// response personalized. + pub asks_for_origin: bool, + /// The URL core asks of the origin, with its query. + pub origin_url: &'a str, + /// What names the origin beyond the URL, so one origin's file is never + /// taken for another's. + pub origin_identity: &'a str, + /// The request core would send to the origin. + pub origin: OriginRequest, +} + +/// The request core would send to the file's origin, a `GET` with no body. +#[derive(Debug, Clone)] +pub struct OriginRequest { + uri: Uri, + headers: HeaderMap, + backend_name: String, + cache_intent: PlatformCacheIntent, + stream_response: bool, +} + +impl OriginRequest { + /// What `request` asks for, being the request core sends itself when the + /// module passes the file back, or `None` when it is not a `GET` with no + /// body. + /// + /// A request that describes a body, by `Content-Length` or + /// `Transfer-Encoding`, is never offered, because a module's load sends + /// it with no body, and a load the origin refuses is the answer to every + /// request waiting on it. + pub(crate) fn of(request: &PlatformHttpRequest) -> Option { + let headers = request.request.headers(); + if request.request.method() != Method::GET + || headers.contains_key(header::CONTENT_LENGTH) + || headers.contains_key(header::TRANSFER_ENCODING) + { + return None; + } + Some(Self { + uri: request.request.uri().clone(), + headers: headers.clone(), + backend_name: request.backend_name.clone(), + cache_intent: request.cache_intent.clone(), + stream_response: request.stream_response, + }) + } + + /// The request core would send, built anew for each call. + #[must_use] + pub fn platform_request(&self) -> PlatformHttpRequest { + let mut request = Request::new(EdgeBody::empty()); + *request.method_mut() = Method::GET; + *request.uri_mut() = self.uri.clone(); + *request.headers_mut() = self.headers.clone(); + let mut platform_request = PlatformHttpRequest::new(request, self.backend_name.clone()); + platform_request.cache_intent = self.cache_intent.clone(); + platform_request.stream_response = self.stream_response; + platform_request + } + + /// The URI the request asks for. + #[must_use] + pub fn uri(&self) -> &Uri { + &self.uri + } + + /// The headers the request carries, which hold the values `Vary` names. + #[must_use] + pub fn headers(&self) -> &HeaderMap { + &self.headers + } +} + +#[cfg(any(test, feature = "test-utils"))] +impl<'a> AssetFile<'a> { + /// A file offered on `via` for `origin_url`, with nothing set that keeps + /// it from a copy, which a module's test changes field by field. + #[must_use] + pub fn for_test( + via: AssetVia, + host: &'a str, + origin_url: &'a str, + origin: OriginRequest, + ) -> Self { + Self { + via, + carries_authorization: false, + carries_cookie: false, + host, + scheme: "https", + is_document: false, + asks_for_origin: false, + origin_url, + origin_identity: origin_url, + origin, + } + } +} + +#[cfg(any(test, feature = "test-utils"))] +impl OriginRequest { + /// A `GET` of `uri` with `headers` to `backend_name`, with the platform's + /// default cache behavior, for a module's tests. + #[must_use] + pub fn for_test( + uri: Uri, + headers: HeaderMap, + backend_name: &str, + stream_response: bool, + ) -> Self { + Self { + uri, + headers, + backend_name: backend_name.to_owned(), + cache_intent: PlatformCacheIntent::Default, + stream_response, + } + } +} + +/// Asset cache modules for core's own tests of the two paths. +#[cfg(test)] +pub(crate) mod test_support { + use std::sync::{Arc, Mutex}; + + use super::*; + + /// What a module was offered, kept beyond the call that offered it. + #[derive(Debug, Clone)] + pub(crate) struct Offered { + pub(crate) via: AssetVia, + pub(crate) carries_authorization: bool, + pub(crate) carries_cookie: bool, + pub(crate) host: String, + pub(crate) scheme: String, + pub(crate) is_document: bool, + pub(crate) asks_for_origin: bool, + pub(crate) origin_url: String, + pub(crate) origin_identity: String, + pub(crate) origin: OriginRequest, + } + + impl Offered { + fn of(file: &AssetFile<'_>) -> Self { + Self { + via: file.via, + carries_authorization: file.carries_authorization, + carries_cookie: file.carries_cookie, + host: file.host.to_owned(), + scheme: file.scheme.to_owned(), + is_document: file.is_document, + asks_for_origin: file.asks_for_origin, + origin_url: file.origin_url.to_owned(), + origin_identity: file.origin_identity.to_owned(), + origin: file.origin.clone(), + } + } + } + + /// How a test module answers every file it is offered. + enum Answering { + AskOrigin, + Serve(fn() -> Response), + OriginFailed, + } + + /// An asset cache module that records each file it is offered and + /// answers every one the same way. + pub(crate) struct TestAssetCache { + answering: Answering, + offered: Mutex>, + } + + impl TestAssetCache { + fn answering(answering: Answering) -> Arc { + Arc::new(Self { + answering, + offered: Mutex::new(Vec::new()), + }) + } + + /// A module that passes every file back to core. + pub(crate) fn asking_the_origin() -> Arc { + Self::answering(Answering::AskOrigin) + } + + /// A module that serves what `response` makes for every file. + pub(crate) fn serving(response: fn() -> Response) -> Arc { + Self::answering(Answering::Serve(response)) + } + + /// A module whose own request to the origin fails for every file. + pub(crate) fn failing_to_reach_the_origin() -> Arc { + Self::answering(Answering::OriginFailed) + } + + /// Every file offered so far, in order. + pub(crate) fn offered(&self) -> Vec { + self.offered.lock().expect("should lock the offers").clone() + } + } + + #[async_trait::async_trait(?Send)] + impl AssetCacheModule for TestAssetCache { + async fn answer(&self, _services: &RuntimeServices, file: AssetFile<'_>) -> AssetAnswer { + self.offered + .lock() + .expect("should lock the offers") + .push(Offered::of(&file)); + match self.answering { + Answering::AskOrigin => AssetAnswer::AskOrigin, + Answering::Serve(response) => AssetAnswer::Serve(response()), + Answering::OriginFailed => { + AssetAnswer::OriginFailed(Report::new(PlatformError::HttpClient)) + } + } + } + } +} diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 09ced573f0..2d30dcbffc 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -1180,6 +1180,30 @@ mod loading_caches_from_tests { assert!(message.contains("robots-txt, answers"), "{message}"); } + #[test] + fn core_declares_only_the_robots_txt_cache() { + assert_eq!( + declared_caches(&[]).expect("core's own declaration is not refused"), + [ROBOTS_TXT_CACHE], + "a cache of the files the origin sends is a module's to declare" + ); + } + + #[test] + fn a_cache_assets_table_is_refused() { + let toml_str = format!( + "{}\n[cache.assets]\nstore_seconds = 3600\n", + crate::test_support::tests::crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse the settings"); + + let refused = loading_caches_from(&settings, &[offering("loading-cache.one")]) + .expect_err("no linked module declares a cache named `assets`"); + + let message = format!("{refused:?}"); + assert!(message.contains("[cache.assets]"), "{message}"); + } + #[test] fn the_deploy_check_refuses_the_table_the_cache_build_refuses() { let settings = settings_declaring("answres", Some("one")); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 528abffe55..55ac9a53b3 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -9,6 +9,7 @@ use http::{Method, Request, Response}; use matchit::Router; use sha2::{Digest as _, Sha256}; +use crate::asset_cache::AssetCacheModule; use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; @@ -684,6 +685,11 @@ pub struct IntegrationRegistration { /// Declaring them does not put them in the file, because the file is only /// built from the contributors `[robots-txt] modules` names. pub robots_txt_contributor: Option<(&'static str, Arc)>, + /// Asset cache module this registration supplies, under its name. + /// + /// Taken only from the registration of the builder `[asset-cache]` + /// selects, and refused from any other. + pub asset_cache_module: Option<(&'static str, Arc)>, } impl IntegrationRegistration { @@ -715,6 +721,7 @@ impl IntegrationRegistrationBuilder { device_module: None, prefix_module: None, robots_txt_contributor: None, + asset_cache_module: None, }, } } @@ -823,6 +830,22 @@ impl IntegrationRegistrationBuilder { self } + /// Declare the asset cache module this registration supplies, under its + /// `name`, which may answer for a file before core asks its origin, see + /// [`crate::asset_cache`]. + /// + /// The registry takes it only when `[asset-cache]` selects the builder + /// that made this registration, and refuses it from any other builder. + #[must_use] + pub fn with_asset_cache_module( + mut self, + name: &'static str, + module: Arc, + ) -> Self { + self.registration.asset_cache_module = Some((name, module)); + self + } + /// Mark this integration's JS module for deferred loading via /// `