Hold the publisher's assets in the loading cache

.cargo/config.toml

@@ -26,10 +26,10 @@ test_details = "test --target aarch64-apple-darwin"
 # native crate needs no change here. Axum (native), Cloudflare
 # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests
 # (native) are simply not listed.
-build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1"
-check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1"
-clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings"
-test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1"
+build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1"
+check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1"
+clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings"
+test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1"
 # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass
 # --all-features, so without this the reusable-sandbox code is linted by
 # `clippy-fastly` but its tests never execute.
@@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip
 # run with the Axum adapter in CI.
 build-axum = "build -p trusted-server-adapter-axum"
 check-axum = "check -p trusted-server-adapter-axum"
-clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings"
-test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb"
+clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings"
+test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb"
 
 # --- Cloudflare adapter (native host + wasm32-unknown-unknown) ---
 # Build/check target the WASM runtime (requires the `cloudflare` feature);

CHANGELOG.md

@@ -100,6 +100,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - `[ad-tag.google] write_loader` writes the loader tag, for a page that carries no loader of its own, through the serve middleware `ad-tag.google.loader`, which a `[[serve]]` entry names and `ts audit` writes where it finds GPT. The tag goes in the head after the bootstrap and asks for the loader on the module's own route. Only a reader whose permissions allow storage on the device (`necessary.operations.storage`) gets it, because GPT loaded from Google's standard address may use the device's storage at any time. That is TCF Purpose 1, which core's server-side auction requires where GDPR applies and which the Google tag module's loaders require. Off by default, because a page that already carries the loader would otherwise load it twice.
+- An asset cache module may answer for a file before core asks its origin. Core offers each `GET` for a file that the publisher's origin sends or an asset route fetches, other than one the platform's image optimizer answers, to the module that `[asset-cache]` selects, with what it knows of the reader's request and the request it would send. The module serves a response, which core handles as it would the origin's, reports that its own origin request failed, which core answers as a failed origin request, or passes the file back, and a deployment that selects none has every file asked of its origin. A registration supplies one with `with_asset_cache_module`, and the registry refuses one from a builder that `[asset-cache]` does not select, an `[asset-cache]` that selects two, and one that selects a builder that supplies none, naming that builder. The new `crates/asset-cache/51degrees`, the module `asset-cache.51degrees`, which no stock build links, holds each file as its origin sent it, under what the origin's `Vary` names and for the origin's own lifetime, in two loading caches it declares, `51degrees-assets` for the copies and `51degrees-assets-vary` for each address's record, each selecting its own loading cache module and both sized by `[cache.51degrees-assets]`. It keeps nothing that may not be shared, serves a copy with an `Age` that counts from the origin's, and on the publisher path leaves a document, and every request that carries a cookie, to the origin. A load that fails asking the origin is the answer to every request waiting on it, so a failing origin is asked once. A table that names `none` is refused, as one naming any module no linked crate offers is.
 - A Google tag module, `[tag] modules = ["google-tag"]` with `[tag.google-tag]`, from `crates/tag/google-tag`, which writes the consent mode defaults, Tag Manager's container script, the gtag.js snippet and the AdSense loader in the head immediately before the script bundle's tag, and Tag Manager's noscript iframe after the opening body tag, each copied from Google's documentation. Every reader gets the consent defaults, and only a reader whose permissions allow storage on the device (`necessary.operations.storage`) gets the loaders and the noscript iframe. The consent defaults come first, the four consent types Google requires for its own services default to denied, the tag waits `consent_defaults.wait_for_update` milliseconds (500 unless set) for a consent management platform's update before it fires with them, and `consent_defaults.region` scopes them to country or region codes, with every other visitor given all four denied. Where the Tag Manager module serves the same tag or container the loader names the first-party path, and a page's own loader for the configured tag is removed for every reader.
 - A middleware may write markup immediately after the opening body tag, with `body_open_inserts` on its action, once per document, beside the places it had for the head and for straight after the script bundle.
 - A creative-opportunity format may name `min_viewport_width` and `max_viewport_width`, and a slot with such a format asks for only the formats that fit each band of viewport widths. The slot's page data carries `viewport_sizes`, and the Google ad tag's head bootstrap and the bundle's slot definition apply it as a size mapping, define no slot where the band asks for no size, and keep every format when the mapping fails. The bundle's refresh auction offers a bidder only the sizes of the band the viewport is in for a slot defined with that mapping, and offers a slot the publisher defined its formats as before. A winning bid with no size of its own takes the band's first size, and `ts config ad-templates --details` prints each format's bounds. A slot whose formats name no bound behaves as before.

Cargo.toml

@@ -22,6 +22,7 @@ members = [
     "crates/ads-txt/sellers",
     "crates/headers/geo",
     "crates/loading-cache/51degrees",
+    "crates/asset-cache/51degrees",
     "crates/tag/google-tag",
     "crates/tag/google-tag-manager",
     "crates/testing/testlight",
@@ -172,6 +173,7 @@ trusted-server-cmp-inmobi = { path = "crates/cmp/inmobi" }
 trusted-server-ads-txt-sellers = { path = "crates/ads-txt/sellers" }
 trusted-server-headers-geo = { path = "crates/headers/geo" }
 trusted-server-loading-cache-51degrees = { path = "crates/loading-cache/51degrees" }
+trusted-server-asset-cache-51degrees = { path = "crates/asset-cache/51degrees" }
 trusted-server-tag-google-tag = { path = "crates/tag/google-tag" }
 trusted-server-tag-google-tag-manager = { path = "crates/tag/google-tag-manager" }
 trusted-server-testing-testlight = { path = "crates/testing/testlight" }

crates/asset-cache/51degrees/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "trusted-server-asset-cache-51degrees"
+description = "An asset cache module that holds the files the publisher's origin and the asset routes send, as each origin sent them, in two loading caches."
+authors = { workspace = true }
+edition = { workspace = true }
+license = { workspace = true }
+publish = { workspace = true }
+version = { workspace = true }
+
+[lib]
+doctest = false
+
+[lints]
+workspace = true
+
+[dependencies]
+trusted-server-core = { workspace = true }
+async-trait = { workspace = true }
+bytes = { workspace = true }
+derive_more = { workspace = true }
+edgezero-core = { workspace = true }
+error-stack = { workspace = true }
+futures = { workspace = true }
+hex = { workspace = true }
+http = { workspace = true }
+log = { workspace = true }
+sha2 = { workspace = true }
+web-time = { workspace = true }
+
+[dev-dependencies]
+serde_json = { workspace = true }
+tokio = { workspace = true }
+trusted-server-core = { workspace = true, features = ["test-utils"] }
+
+# The visible owner of this vendor crate, the way Prebid.js requires a named
+# maintainer of every adapter.
+[package.metadata.maintainers]
+owner = "51Degrees"
+status = "vendor owned"