diff --git a/.cargo/config.toml b/.cargo/config.toml index 877af07db0..385aea1623 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -28,8 +28,8 @@ test_details = "test --target aarch64-apple-darwin" # (native) are simply not listed. build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-shared-aws -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-shared-aws -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-shared-aws -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-shared-aws -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-provenance-encypher -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-shared-aws -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-provenance-encypher -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-shared-aws -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-loading-cache-51degrees -p trusted-server-shared-aws -p trusted-server-shared-51degrees -p trusted-server-middleware-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-loading-cache-51degrees -p trusted-server-shared-aws -p trusted-server-shared-51degrees -p trusted-server-middleware-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-provenance-encypher -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-loading-cache-51degrees -p trusted-server-shared-aws -p trusted-server-shared-51degrees -p trusted-server-middleware-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-provenance-encypher -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-loading-cache-51degrees -p trusted-server-shared-aws -p trusted-server-shared-51degrees -p trusted-server-middleware-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index 37560cd725..2e1c1f6ac6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Encypher can sign bounded media with source ingredients and executed transformation actions. The opt-in large-media lane hashes a version-pinned origin locally and uses prepared signing without uploading or buffering the complete asset. It retains existing embedded, XMP-referenced and HTTP-linked provenance rather than publishing a competing manifest. Enable a bounded `max_large_media_bytes` only after the matching signed probe and backend release are available and the adapter can enforce the whole-operation deadline; the default is `0`. Fastly has no total timer and keeps this lane closed. +- Encypher's `max_media_sign_calls_per_document` limits uncached embedded-media signing calls per HTML or CSS document. It defaults to 8 and accepts 1-128. Cache hits do not consume calls; exceeding the limit leaves the remaining media unchanged without negatively caching it. Raising the limit does not extend the document's time or output-size bounds. +- Large-media namespace resolution, cache lookup and new hash/sign work share one remaining-time budget. An already exhausted budget starts no hashing. Malformed `article_selector` overrides now fail settings admission instead of silently choosing the normal article boundary; empty and valid-but-absent overrides keep that fallback. +- A declined large-media sign returns one fresh, private/no-store origin response with its own validators, redirects, authentication challenges and retry delay. Publisher redirect targets use the existing serving-host mapping; the fallback does not retry or follow redirects. +- With a media signer installed, an Image Optimizer route asks Fastly Image Optimizer to keep its input's C2PA manifest and add its own record of the transformations it performed (`metadata=c2pa`). An optimized answer that carries a C2PA manifest store is signed as opened, with Image Optimizer's manifest as its parent and `Vary: Accept`; one without stays unsigned as before. A route that sets `[proxy.asset_routes.image_optimizer] signed_source = true` sends Image Optimizer to its own signed original on the serving host, making eligible unsigned originals available as signed inputs at the cost of a second request per image, and answers once from the origin without Image Optimizer when Image Optimizer fails. Deployed Image Optimizer behavior has not been verified. +- Media signing leaves the parts of a fragmented ISO BMFF stream alone, so a live or DASH/HLS stream part within the signer's bounded limit, whose provenance its init segment binds, reaches the reader byte for byte with the origin's own headers and no competing manifest, whatever its `Content-Type` or `Cache-Control`. Core reads the bytes it already captured: a media segment starts with `styp` or has a `moof` before its first `mdat` and has no `moov`, and an init segment starts with `ftyp` and has a `moov` holding an `mvex` and no `mdat`. A progressive MP4 is signed as before. Encypher also never claims `video/iso.segment`. +- An asset route that sets `rehost_page_references = true` serves a third party's public files from the publisher's host, making eligible images, fonts and embedded stylesheet media available for signing. Pages served through the publisher pipeline name the route's address for media references (`src`, `srcset` and `imagesrcset` candidates, `video` `poster`, media preloads) and, where a `[[serve]]` entry for `text/css` covers the path, stylesheet references, but only where request dispatch reaches that route and the route fetches exactly the address the page wrote. An element with `integrity` keeps its address. The route sends no `Referer` and asks for an identity-coded body. A moved stylesheet has each address it names resolved against its upstream address and written as a route address or the absolute upstream address, then runs through its `[[serve]]` entry, without the origin's validators. A stylesheet that cannot be moved, for a status other than `200`, a coded body, a `private` or `no-transform` answer, a charset other than UTF-8, a body over 10 MiB or a middleware refusal, is answered with an uncached `307` to its upstream address. Redirects are written at the route's address where one serves their target. On a path a stylesheet entry covers, the reader's conditions are answered against what is delivered. Settings refuse a rehosting route with path rewriting, Image Optimizer, origin auth, a foreign `Host`, a directory index, a stripped query, a prefix under `/_ts` or one overlapping another route. +- The CSS address rewriter creatives use is shared with rehosted stylesheets, and media signing answers `If-Modified-Since` and `If-Unmodified-Since` as well as entity-tag conditions on the answers it delivers whole. +- Rehosting resolves URLs against the first effective base in the complete decoded original HTML, including references before that base and protocol-relative scheme selection. Rehosting and applicable middleware source-protection policies hold original HTML under the existing publisher body limit. HTML references are decoded once, and `srcset`/`imagesrcset` publisher fallback replaces only a leading authority or bounded bare host, preserving publisher names in other URL components. CSS-chain candidates bypass raw-copy caching; final CSS freshness is capped by remaining origin freshness and route policy after middleware. Empty rewritten redirects discard origin representation headers, and transparent routes validate the complete HTTP Host authority, including non-default ports. +- Encypher signs an inline `` as the standalone SVG document whose XML tree is the tree the browser's HTML parser built: the original bytes where they already parse as XML to that tree, otherwise that tree serialized as XML, kept only where the serialization parses back to the same tree both as HTML and as XML. Before any embedded-data edits, it protects every original provenance-bearing SVG and all of its descendants byte for byte, without submitting either the whole SVG or its children for signing (`svg_existing_provenance`). Uncertain or unclosed protected ranges stay protected through EOF. XML serialization enforces its output bound before every append and namespace insertion. The skip reason `inline_svg_not_standalone` is now `inline_svg_not_equivalent`; `svg_provenance_not_equivalent` remains the equivalence refusal for provenance-bearing input that cannot be preserved as written. +- Applicable middleware can protect exact original source ranges before core, fetch or serve changes. Encypher uses the same SVG classifier as its signing scan to protect provenance-bearing and structurally uncertain SVGs, including late carriers and descendants, even with no rehost routes. The bounded HTML pass holds opaque, collision-checked markers, restores original byte slices before publishing a shared template, and protects the reader copy again on cache replay. Lost, duplicated, reordered or altered markers, invalid classifier ranges, and input/masked/restored output limit excess refuse the pass rather than serving altered signed SVG. Unsigned SVG normalization/signing and surrounding page transformations remain enabled. +- Moved CSS keeps browser `max-age` separate from shared `s-maxage` (falling back to `max-age`). It subtracts the greater of received `Age` and apparent age from `Date`, plus processing time, before rebasing headers, then applies the corresponding browser and shared route caps separately. Streamed stylesheet collection uses the existing collector's pre-append capacity check; buffered bytes remain zero-copy. +- On a host that serves several publishers by path, through a deployment's request addressing, Encypher signs each article, embedded item and media file for the address the reader asked for, with the publisher's path prefix, where it named the address without the prefix, so two publishers' pages at one path no longer share an address. The path exclusions still read the publisher's own path. Module middleware sees the removed prefix on the publisher path as `ModuleRequest::removed_prefix`. +- The Axum, Cloudflare and Spin HTTP clients stream an origin response when the caller asks for one, as the Fastly client does, so `supports_streaming_responses` is true on every adapter. A streamed body is read one chunk at a time and never held whole. A caller's `max_response_body_bytes` is enforced exactly, chunk by chunk, in place of the buffered ceilings of 10 MiB on Cloudflare and 8 MiB on Spin. A declared `Content-Length` over the limit is refused before any byte is read, and an origin read failure, or a body cut short of its declared length on Cloudflare and Spin, ends the stream with an error instead of a clean end. A Spin streamed body keeps the origin's `Content-Encoding`, and the Workers runtime still decodes one. `send_async` stays buffered on these adapters and refuses a request that asks for a stream. Delivery streams too. The Axum dev server runs its own front end in place of `AxumDevServer`, whose service collected each streamed response into one buffer: it passes chunks to Hyper through a bounded channel, stops the origin read when the client leaves, and runs at most 256 requests at once. The Spin entry point dispatches through the router itself in place of `run_app`, whose conversion collected a streamed response up to 16 MiB, and hands the host one frame per chunk. +- Provenance coverage is limited to eligible responses and source-embedded media processed by configured routes and middleware. Direct third-party requests and script-created SVG remain outside that coverage. Local real-API and browser verification does not establish complete page coverage, public detached-manifest retrieval or a production deployment. - The Tinybird project, its datasources, pipes, fixtures and tests, moves from `tinybird/` to `crates/analytics/tinybird/project/`, beside the analytics module's Rust, so run `tb` from the new folder. - A vendor module's browser script and its tests live in the crate that supplies the module, under `js/src//` and `js/test//`, where they were under `crates/trusted-server-js/lib/src/integrations/` and `lib/test/integrations/`, for DataDome, Didomi, Google Tag Manager, GPT, GPT diagnostics, Lockr, Osano, Permutive, Sourcepoint and Testlight. The built bundles are unchanged. A script in a crate imports the shared browser code as `@tsjs/core/...` and `@tsjs/shared/...`, and another module's code as `@tsjs/module//...`. -- The Axum, Cloudflare and Spin adapters serve `[[proxy.asset_routes]]`, where only the Fastly adapter did, so a `GET` or `HEAD` under a route's prefix is answered from the route's source, with its signing and the answers held for it, and no longer by the publisher's origin. The response skips the Edge Cookie steps, so it neither writes nor expires the Edge Cookie, and takes the route's cache policy after the finalize headers. Those adapters hold the asset body whole, up to 10 MiB on Cloudflare and 8 MiB on Spin. Their HTTP clients refuse Image Optimizer metadata, which the Axum client now does on both of its send paths, so there a route that enables the Image Optimizer answers each request it would optimize with `502` and `no-store, private`, after a signed route has asked its origin for the object with a `HEAD`, and serves its SVG paths and requests that set the debug parameter unoptimized. Each adapter logs a warning naming every such route, once for each instance of the adapter, which on Axum is the process. +- The Axum, Cloudflare and Spin adapters serve `[[proxy.asset_routes]]`, where only the Fastly adapter did, so a `GET` or `HEAD` under a route's prefix is answered from the route's source, with its signing and the answers held for it, and no longer by the publisher's origin. The response skips the Edge Cookie steps, so it neither writes nor expires the Edge Cookie, and takes the route's cache policy after the finalize headers. Those adapters stream the asset body as the Fastly adapter does, and hold a buffered one whole, up to 10 MiB on Cloudflare and 8 MiB on Spin. Their HTTP clients refuse Image Optimizer metadata, which the Axum client now does on both of its send paths, so there a route that enables the Image Optimizer answers each request it would optimize with `502` and `no-store, private`, after a signed route has asked its origin for the object with a `HEAD`, and serves its SVG paths and requests that set the debug parameter unoptimized. Each adapter logs a warning naming every such route, once for each instance of the adapter, which on Axum is the process. - **Breaking:** on Spin, a request to a source whose `host_header` names a host other than the one in its `url` answers `502` before its origin is asked, the publisher's `[source.origin]` as well as an asset route's source, because Spin sends the host in `url` as the `Host`. Before, the request reached its origin with the host in `url` in place of `host_header`. A Spin deployment whose publisher source sets such a `host_header` answers every page with `502`, robots.txt included. Before upgrading, check that no source of a Spin deployment sets a `host_header` naming another host, or remove it once its origin answers requests that carry the host in `url` as the `Host`. - The Edge Cookie is written without an identity graph as it is with one, because the graph is one optional store of the data kept with the cookie, such as partner IDs. A Fastly service with no `[ec] ec_store` now writes the cookie a real browser's page navigation creates, and `POST /_ts/api/v1/ec/resolve` answers there as it does with a graph, where neither wrote one. So the identifier auction providers get as `user.id`, where storage and personalized-ad selection are both set, is one value for the life of the cookie, where a server-side module such as `hmac` gave a new value on each page view and a client-side module gave none. Naming an `ec_store` later gives each browser holding a cookie a server-side module wrote without one a new identifier on its first page view, because the identity graph replaces an identifier it has no row for. A client-side module creates nothing at the edge to replace its cookie with, so that cookie keeps its identifier with no row for as long as the browser keeps the cookie and its resolved marker, and each of that browser's page views pays for an extra identity-graph read and a strongly consistent existence check. **Breaking for crates outside core:** `ec_finalize_response` and `handle_ec_resolve` read the bot gate from the context's device signals (`EcContext::set_device_signals`), so a context with none gets no created cookie. - The Axum, Cloudflare and Spin adapters create and write the Edge Cookie and serve `POST /_ts/api/v1/ec/resolve`, where only the Fastly adapter did. A real browser's page navigation is given an Edge Cookie where its permissions allow one, and every route in the request cycle finalizes the Edge Cookie once the response is finalized, which writes a created cookie under the request's public prefix and path and expires one on a withdrawal. Those adapters open no identity graph, so the cookie is written with no graph row, by the resolve route too, the graph's syncs are skipped, and startup logs once that the store `[ec] ec_store` names is not opened. The bot gate there passes any client that sends a browser's `User-Agent`, and where storage and personalized-ad selection are both set a real browser's identifier reaches auction providers as `user.id`, from its first page view with a server-side module and from the requests after its resolve with a client-side one. @@ -49,7 +64,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The Axum dev server and Spin, which have no geo lookup, refuse `[geo] module = "platform"` at startup naming the setting. The selection located no request there and still counted as a geo module, so a deployment running an Edge Cookie module was never asked to set `[geo] assume_single_jurisdiction`. The configuration and Cloudflare guides say that Cloudflare places a visitor in a region only from the `cf-region-code` request header. - A module declares the caches it loads through, by name, and a deployment selects a loading cache for each in its own `[cache.]` table, the way `[cache.robots-txt]` selects one for robots.txt. A `[cache.]` table naming a cache no linked module declares is refused by `ts config validate` and when the deployment starts, naming the ones it does, and a module declaring `robots-txt` or a name another module declares is refused. `IntegrationBuilder::with_cache_features` declares them, `CacheSettings` has a new public field `declared`, `CacheSettings::loading_caches` returns a `Vec` in place of an array, listing core's own and the declared ones, and `LoadingCaches::insert` takes any name. - **Breaking for crates outside core:** the cookie constants `COOKIE_TS_EC`, `COOKIE_TS_EC_RESOLVED`, `COOKIE_TS_EC_PULL_COMPLETE`, `COOKIE_TS_EIDS` and `COOKIE_TS_TESTER` are `COOKIE_EC`, `COOKIE_EC_RESOLVED`, `COOKIE_EC_PULL_COMPLETE`, `COOKIE_EIDS` and `COOKIE_TESTER`, holding the entry's name alone, and a cookie is named through the request's public prefix with `services.public_prefix().cookie_name(COOKIE_EC)`. A module's response header naming a cookie under the public prefix is refused as one under `ts-` was. -- A robots.txt module's answer is held by the loading cache of the one loading cache module the build links, or of the module `[cache.robots-txt] module` names, briefly in memory and in the host's shared store outside it, so a module is asked again only when no copy young enough is held, rather than by every request. With no loading cache module linked, or several and none named, nothing is held and every request for the file asks each module, as every request did when no adapter gave the file a key-value store. The last answer is kept apart for 30 days and served when the module cannot answer, in the value cache on Fastly and in the key-value store on Cloudflare and Spin, and Axum, which has no key-value store, keeps none. +- A robots.txt module's answer is held by the loading cache of the one loading cache module the build links, or of the module `[cache.robots-txt] module` names, briefly in memory and in the host's shared store outside it, so a module is asked again only when no copy young enough is held, rather than by every request. With no loading cache module linked, or several and none named, nothing is held and every request for the file asks each module, as every request did when no adapter gave the file a key-value store. The last answer is kept apart for 30 days and served when the module cannot answer, in the value cache on Fastly and in the key-value store on Cloudflare and Spin. Axum persists it in the redb file named by `TRUSTED_SERVER_KV_PATH`, shared with Encypher and other runtime KV consumers; with that variable unset, Axum keeps no persistent fallback. - **Breaking:** the Cloudflare and Spin adapters declare their key-value store, the Workers KV binding `TRUSTED_SERVER_KV` and Spin's `default` store, so EdgeZero opens it for every request and the loading cache keeps its copies there between requests, where every request was given a store whose every call failed. A declared store is required, so a Worker without the binding, or a Spin component that cannot open `default`, fails every request before any route runs. - A consent management platform's middleware runs in the phase the publisher chooses. `cmp.inmobi`, `cmp.didomi` and `cmp.sourcepoint` may be named in a `[[fetch]]` or a `[[serve]]` entry, where each ran in the fetch phase alone. A generated configuration still places them in `[[fetch]]`. A `[[serve]]` entry naming `cmp.inmobi` for a page whose `[[fetch]]` entry names another middleware is refused, because the serve phase writes its head markup after what the fetch phase writes ahead of the script bundle, so a vendor's tag there would come ahead of the prompt. Naming `cmp.inmobi` for one page in both phases is refused, because the prompt would be written twice. - **Breaking:** auction telemetry is selected the way every other module is. `[tinybird]` is now `[analytics.tinybird]`, and `[analytics] module = "tinybird"` selects it, which replaces the `enabled` line. A configuration that still carries `[tinybird]` is refused with directions. A stored configuration written by an earlier version carries the table even where telemetry was off, so push the configuration again with this version. The secret reference is `analytics.tinybird.auction_token_secret`, the table refuses a key it does not know, and a section that selects nothing, or a module this build does not have, is refused. A row is sent only for a reader whose permissions include `analytics.ad_reporting.measure_ad_performance`, so none is sent for a reader the consent gate refuses unless they permit it. The Axum, Cloudflare and Spin adapters send no telemetry, and log a warning at startup when the section selects a module. @@ -109,6 +124,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Encypher now applies `max_document_bytes` to the exact signed HTML size before allocating output, including every marker and the escaped manifest meta element. An API plan that would expand a page past the publisher buffer limit now leaves the original page unchanged instead of making buffered delivery fail. Policy and signing calls bypass platform intermediary caches and carry a receive-time body bound, enforced chunk by chunk by each adapter before it buffers or yields the crossing chunk. +- Encypher stores positive plans and policy digests separately from short-lived failures. A delayed failed request can no longer replace a concurrent successful result. Malformed and structurally invalid cache entries are deleted best effort so the next view can recover, while a valid plan that does not align with one HTML representation remains available to other representations. - With `consent_forwarding = "openrtb_only"`, the Model Terms for Marketing preference cookie, `__mtm_pref`, is withheld from Prebid Server along with the consent cookies. It was forwarded in every mode. - A response no longer carries `X-Served-By`, `X-Cache` or `X-Cache-Hits`, unless this server answered a request for a path beneath `/_ts`. The three name the node that answered and say how a cache treated the request, and arrive from the publisher's origin or a cache on the way to it. Every adapter removes them before it finalizes a response, so one set in `[response_headers]` is still sent. - No adapter writes a version on a response. What the build was made from is read from the attestation document, under `build`, and the platform's version number of the service under `platform.serviceVersion`, and a deployment that wants them on its pages names core's `service_version` middleware in a `[[serve]]` entry. @@ -135,6 +152,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Encypher article provenance is available on Fastly, Cloudflare, Spin and Axum. Select `[provenance] module = "encypher"` and place `provenance.encypher` in a `[[serve]]` entry for public article routes. Signing failures pass the held page through unchanged; origin-private and Set-Cookie responses are excluded. Signing requires a runtime KV store: link Fastly's `trusted_server_kv`, bind Cloudflare's `TRUSTED_SERVER_KV`, declare Spin's `default` store, or set Axum's `TRUSTED_SERVER_KV_PATH`. See the [Fastly deployment prerequisites](docs/guide/fastly.md) and [Spin storage and retention guidance](docs/guide/spin.md). - `ts config migrate --adapter ` rewrites a settings document written for an earlier release, such as one written before the breaking changes above, into the shape this release reads, and prints the result. The host's location lookup the earlier release ran becomes `[geo] module = "platform"` on Fastly and Cloudflare and no geo module on Axum and Spin, which have no lookup. On stderr it lists each change it made, each setting it left for the operator because it needs a decision or a value the document does not hold, such as `[geo] assume_single_jurisdiction` for an Edge Cookie module with no geo module, and each change in behavior no setting expresses, then checks the result with the settings validation `ts config validate` runs. It keeps the comments and the order of the tables, comments out a module that was not enabled under its new name, never selects `headers.geo`, which writes the reader's location, never prints a value from the document, rewrites the file in place with `--write`, and with `--check` changes nothing and exits 1 when the document needs migrating. Migrating a migrated document changes nothing. - A 51Degrees client hints middleware, `51degrees.client_hints`, from `crates/middleware/51degrees`, run by `[shared.51degrees]` and placed by a `[[serve]]` entry. It asks the 51Degrees service which client hints the reader's browser can send for each page navigation an entry covers, sending the `User-Agent` and `Sec-CH-UA` hints alone, and leaves the hint names in the request's state for the other 51Degrees modules, which keeps a page it names hints for out of the stored templates and sends it `private, no-store`. On each page an entry covers it writes `Accept-CH` and `Critical-CH` on the response unless the browser already sends every one of those hints, `Permissions-Policy` and the `Delegate-CH` tag that delegate those hints to the service's origin, each keeping every line the response already carries. `critical` in `[middleware.51degrees.client_hints]` turns the retry of the first navigation off. The service's answers are held in the cache `51degrees-answers`, selected and sized in `[cache.51degrees-answers]`. What the 51Degrees modules share, being the table, the one call to the service, the evidence it sends and the cache its answer is held in, is the new `crates/shared/51degrees`. - `[middleware.]`, a middleware's own settings table, written with the middleware's full name and read through `Settings::middleware_config`. A table holds values and arrays, a table within one is the longer name's table and never a setting, and a table at a name no running module supplies is refused at startup with the names that could be written. @@ -151,7 +169,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `robots_txt`, core's own middleware for the file `[robots-txt]` describes, named in a `[[fetch]]` entry for `text/plain` on `/robots.txt`. Its preparation makes the file from the selected modules through their holding, and its action replaces the origin's file with it. The file is made in place of any other answer the origin gives to a `GET` or `HEAD`, such as a `404`, a page, an error or a redirect. The origin is asked for its whole file, without a crawler's conditional or range headers, and the middleware works on `/robots.txt` alone. A module that cannot answer with nothing held refuses the request with `503` and `Retry-After` before any document is processed. A section with no such entry, or with an entry naming the middleware on another path or media type, is refused when the settings load. - A middleware may say which request paths it works on, with `Middleware::handles_path`, that it writes the whole document, with `writes_whole_document`, so core makes the document in place of any answer it would not hand the middleware as a `200` of the entry's media type, and that it needs the origin's whole body, with `needs_whole_body`, so the origin is asked without the reader's conditional and range headers. By default a middleware works on every path its entry covers and says neither of the other two. - A middleware may prepare, and may refuse. `Middleware::prepare` runs in the request's handler before the document is processed, as the middleware's module with the request's services, and what it answers is kept under the phase and the middleware's name for `create` in that phase, so a decision that needs a store or the network is made there while `create` stays synchronous. The fetch phase of a document that may be stored is prepared with no client IP, TLS details or client signals in the services. A refusal answers the request with its status and `Retry-After`, with no document, and nothing is stored. Core logs a refusal as a warning naming the middleware, and answers a status that is not a client or server error with `500`. -- A loading cache. `PlatformLoadingCache` in core loads a missing value once per key however many requests ask for it at the same moment, gives every waiting request the value or the error, and holds nothing that failed. A load is handed services with nothing of the reader in them, from `RuntimeServices::without_connection`, because its value serves every reader. A crate offers one through `IntegrationBuilder::with_loading_cache`, each feature that holds answers selects the module whose cache holds them in its own `[cache.]` table, and an adapter takes the selections with `loading_caches_from`. A feature that selects none takes the one loading cache module the build links, and gets `NoLoadingCache`, which loads every time, when the build links none or several. The new `crates/loading-cache/51degrees`, the module `loading-cache.51degrees`, implements it with the published `fiftyone-caching` package as a brief copy in memory over the host's value cache on Fastly or its key-value store on Cloudflare and Spin, with the memory copy alone on Axum, and the stock list ships it, so every adapter takes it. Each application state keeps a memory copy of its own, which lasts one request where an adapter builds its state for each request, as Cloudflare and Spin do. Every load runs in its caller. +- A loading cache. `PlatformLoadingCache` in core loads a missing value once per key however many requests ask for it at the same moment, gives every waiting request the value or the error, and holds nothing that failed. A load is handed services with nothing of the reader in them, from `RuntimeServices::without_connection`, because its value serves every reader. A crate offers one through `IntegrationBuilder::with_loading_cache`, each feature that holds answers selects the module whose cache holds them in its own `[cache.]` table, and an adapter takes the selections with `loading_caches_from`. A feature that selects none takes the one loading cache module the build links, and gets `NoLoadingCache`, which loads every time, when the build links none or several. The new `crates/loading-cache/51degrees`, the module `loading-cache.51degrees`, implements it with the published `fiftyone-caching` package as a brief copy in memory over the host's value cache on Fastly or its key-value store on Cloudflare and Spin. Axum uses the persistent runtime KV file when `TRUSTED_SERVER_KV_PATH` is set and only the memory copy when unset. The stock list ships the module, so every adapter takes it. Each application state keeps a memory copy of its own, which lasts one request where an adapter builds its state for each request, as Cloudflare and Spin do. Every load runs in its caller. - `[cache.robots-txt]` selects, with `module`, the loading cache that holds each robots.txt module's answer, and sets the lifetimes of its copies, in seconds: `memory_seconds` (default 60), `store_seconds` (default 604800) and `store_idle_seconds` (default 2592000). A memory lifetime may be 0, the others must be at least 1, and none may be more than a year. A `module` no linked crate offers a cache from is refused by `ts config validate` and when the service starts. - `ServerKeyPrefix`, the prefix every loading cache key starts with, being `ts.` and the publisher's domain with a leading `www.` dropped, so publishers sharing a store never share an entry. - `PlatformValueCache`, the host's shared cache of small values with lifetimes, which `RuntimeServices` carries where the host supplies one. `lookup_or_reserve` obliges the caller that finds a key missing to fill it where the host can make other callers wait, and `remove` drops an entry. The Fastly adapter supplies one over Fastly's cache, answering the reservation with a transaction lookup and the removal with a purge of the entry's own surrogate key. diff --git a/Cargo.lock b/Cargo.lock index 89550f2f36..e768cc4b3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4814,7 +4814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -4834,7 +4834,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.118", @@ -4847,7 +4847,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.118", @@ -5251,6 +5251,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "roxmltree" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9da7f59328b09405b181fee1018925fad50f157afb43c0c0f5c522c256dd5907" +dependencies = [ + "memchr", +] + [[package]] name = "rpassword" version = "7.5.4" @@ -5960,6 +5969,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "string-interner" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23de088478b31c349c9ba67816fa55d9355232d63c3afea8bf513e31f0f1d2c0" +dependencies = [ + "hashbrown 0.15.5", + "serde", +] + [[package]] name = "string_cache" version = "0.8.9" @@ -6594,9 +6613,11 @@ dependencies = [ name = "trusted-server-adapter-axum" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "axum", "base64", + "bytes", "edgezero-adapter-axum", "edgezero-core", "error-stack", @@ -6606,6 +6627,7 @@ dependencies = [ "serde_json", "simple_logger", "temp-env", + "tempfile", "tokio", "tower 0.4.13", "trusted-server-ad-server-iabtl", @@ -6633,6 +6655,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6644,6 +6667,7 @@ dependencies = [ name = "trusted-server-adapter-cloudflare" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "base64", "bytes", @@ -6683,6 +6707,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6739,6 +6764,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6760,6 +6786,7 @@ dependencies = [ "error-stack", "flate2", "futures", + "http-body", "http-body-util", "log", "serde_json", @@ -6791,6 +6818,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -7017,6 +7045,7 @@ dependencies = [ "trusted-server-identity-lockr", "trusted-server-loading-cache-51degrees", "trusted-server-middleware-51degrees", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -7110,6 +7139,7 @@ dependencies = [ "glob", "hex", "hmac", + "html5ever", "http", "httpdate", "jose-jwk", @@ -7349,6 +7379,34 @@ dependencies = [ "trusted-server-permission-signal-gpc", ] +[[package]] +name = "trusted-server-provenance-encypher" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64", + "bytes", + "ed25519-dalek", + "edgezero-core", + "ego-tree", + "error-stack", + "futures", + "html5ever", + "http", + "log", + "roxmltree", + "scraper", + "serde", + "serde_json", + "sha2 0.10.9", + "trusted-server-core", + "url", + "validator", + "wasmi", + "wat", + "web-time", +] + [[package]] name = "trusted-server-shared-51degrees" version = "0.1.0" @@ -7811,7 +7869,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b6733b8b91d010a6ac5b0fb237dc46a19650bc4c67db66857e2e787d437204" dependencies = [ "leb128fmt", - "wasmparser", + "wasmparser 0.247.0", +] + +[[package]] +name = "wasm-encoder" +version = "0.262.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4495a6f85dd94c411c749303efcbc5defade0b0cbdbc0541495241ab917a9b5a" +dependencies = [ + "leb128fmt", + "wasmparser 0.262.0", ] [[package]] @@ -7822,8 +7890,8 @@ checksum = "665fe59e56cc9b419ca6fcca56673e3421d1a5011e3b65caf6b726fd9e041d10" dependencies = [ "anyhow", "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", + "wasm-encoder 0.247.0", + "wasmparser 0.247.0", ] [[package]] @@ -7839,6 +7907,56 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78693fcdd618e0fc34af59c6b8efa9ac5d58c68df940beff4bedddb6acfe7c27" +dependencies = [ + "spin", + "wasmi_collections", + "wasmi_core", + "wasmi_ir", + "wasmparser 0.228.0", +] + +[[package]] +name = "wasmi_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a8be2aa467cf2d29e96ff759472c36eeb44a3c81c67fc9cb76c9a24c519c557" +dependencies = [ + "string-interner", +] + +[[package]] +name = "wasmi_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69372d5fda3ea3d1e0aa6603c7888110e0187e88ea17cd8fc2e2df0a0e1f37fa" +dependencies = [ + "libm", +] + +[[package]] +name = "wasmi_ir" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f17b774caa13c618c7244f1ee51fe23c5e7b8538a471fa46d9949779758aed6" +dependencies = [ + "wasmi_core", +] + +[[package]] +name = "wasmparser" +version = "0.228.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4abf1132c1fdf747d56bbc1bb52152400c70f336870f968b85e89ea422198ae3" +dependencies = [ + "bitflags 2.13.0", + "indexmap 2.14.0", +] + [[package]] name = "wasmparser" version = "0.247.0" @@ -7851,6 +7969,39 @@ dependencies = [ "semver", ] +[[package]] +name = "wasmparser" +version = "0.262.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "974d643b125b272607e2fff8012ea14a8328acd7621e4bafa7afb32500052b1b" +dependencies = [ + "bitflags 2.13.0", + "indexmap 2.14.0", + "semver", +] + +[[package]] +name = "wast" +version = "262.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30612a0c29249b5f185888a107e8a519d920226b7d07f9d351fd094e9f75eb23" +dependencies = [ + "bumpalo", + "leb128fmt", + "memchr", + "unicode-width", + "wasm-encoder 0.262.0", +] + +[[package]] +name = "wat" +version = "1.262.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228027f2b07bbaf946980ebbe2437fca73be6f54c40c2a8d4f42f277075f8d1" +dependencies = [ + "wast", +] + [[package]] name = "web-sys" version = "0.3.103" @@ -8261,9 +8412,9 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "wasm-encoder", + "wasm-encoder 0.247.0", "wasm-metadata", - "wasmparser", + "wasmparser 0.247.0", "wit-parser", ] @@ -8283,7 +8434,7 @@ dependencies = [ "serde_derive", "serde_json", "unicode-xid", - "wasmparser", + "wasmparser 0.247.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4f1e312212..4904ee8122 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "crates/ads-txt/iabtl", "crates/analytics/tinybird", "crates/headers/geo", + "crates/provenance/encypher", "crates/loading-cache/51degrees", "crates/shared/aws", "crates/shared/51degrees", @@ -98,6 +99,8 @@ edgezero-adapter-fastly = { git = "https://github.com/IABTechLab/edgezero", tag edgezero-adapter-spin = { git = "https://github.com/IABTechLab/edgezero", tag = "v0.0.9", default-features = false } edgezero-cli = { git = "https://github.com/IABTechLab/edgezero", tag = "v0.0.9" } edgezero-core = { git = "https://github.com/IABTechLab/edgezero", tag = "v0.0.9", default-features = false } +# The tree scraper's HTML sink builds; 0.10 is the version scraper locks. +ego-tree = "0.10" env_logger = "0.11" error-stack = "0.6" esi = "0.7.2" @@ -115,8 +118,12 @@ glob = "0.3" getrandom = "0.2" hex = "0.4.3" hmac = "0.12.1" +# The HTML parser scraper drives; the Encypher module parses inline SVG +# fragments with it directly. 0.35 is the version scraper already locks. +html5ever = "0.35" http = "1.4.0" httpdate = "1.0.3" +http-body = "1" http-body-util = "0.1" hyper = "1" hyper-util = "0.1" @@ -136,6 +143,8 @@ rand = "0.8" rcgen = { version = "0.13", features = ["x509-parser"] } regex = "1.12.3" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +# The Encypher module's XML parser for inline SVG; DTDs stay refused. +roxmltree = { version = "0.21.1", default-features = false, features = ["std"] } rpassword = "7.5" rustls = "0.23" rustls-pemfile = "2" @@ -176,6 +185,7 @@ trusted-server-cmp-inmobi = { path = "crates/cmp/inmobi" } trusted-server-ads-txt-iabtl = { path = "crates/ads-txt/iabtl" } trusted-server-analytics-tinybird = { path = "crates/analytics/tinybird" } trusted-server-headers-geo = { path = "crates/headers/geo" } +trusted-server-provenance-encypher = { path = "crates/provenance/encypher" } trusted-server-loading-cache-51degrees = { path = "crates/loading-cache/51degrees" } trusted-server-shared-aws = { path = "crates/shared/aws" } trusted-server-shared-51degrees = { path = "crates/shared/51degrees" } @@ -197,6 +207,10 @@ url = "2.5.8" urlencoding = "2.1" uuid = { version = "1.18", features = ["v4"] } validator = { version = "0.20", features = ["derive"] } +# The Encypher provenance module's probe host: an interpreter with no JIT, +# so the signed probe release runs on every edge target. +wasmi = { version = "2.0", default-features = false, features = ["std", "validate", "auto-dispatch"] } +wat = "1.247" web-time = "1" webpki-roots = "1" which = "8" diff --git a/PRDs/ARCHIVE/trusted-server-v2-encypher.md b/PRDs/ARCHIVE/trusted-server-v2-encypher.md new file mode 100644 index 0000000000..0252d17670 --- /dev/null +++ b/PRDs/ARCHIVE/trusted-server-v2-encypher.md @@ -0,0 +1,263 @@ +# Trusted Server v2: Encypher provenance as a module + +## Status + +Internal completion review cleared at bar 9.5 in cycle 2. Review corrections +remain local on `encypher/v2-provenance-module`, based on +`feature/55-scripts-in-crates` at `5be114773`. +The existing draft https://github.com/IABTechLab/trusted-server/pull/1448 +still contains the earlier published revision, not these corrections. +Successful live API signing remains unverified; in-process publisher signing +passes against a controlled API. Clearance covers the implementation contract, +not deployment admission or live demo readiness. + +## Current Goal + +Internal review complete. Preserve the publication hold and hand off the local +corrections and separate live-signing verification prerequisite. No landing, +deployment, comment or email was performed during this review. + +## Review Loop State + +bar: 9.5 max-cycles: 5 worktree: /home/agentdesk/code/trusted-server-worktrees/v2-provenance +resumed at: Phase 5 (finalization sweeps + completion gate B) +plan gate: NOT PERFORMED. Gate A was never run on this PRD. It is not +retroactively cleared and must not be recorded as cleared. +completion gate: CLEARED (cycle 2). +Cycle 1 NOT CLEARED: +Scores (GPT-5.5 / Opus): correctness 9.0 / 9.1; simplification 9.0 / 9.0; +security 10.0 / 9.6. Both reviewers accepted all 38 CI commands passing in +run `1792869014289`. Medium findings were stale Axum persistence wording +and whole-article canonical/fragment allocation beyond the signed prefix. +Cycle 2 CLEARED: correctness 10.0 / 9.5; simplification 10.0 / 9.6; +security 10.0 / 9.6. Every verdict is pass. Final full gate `1792871494395` +passed all 38 commands; final rebuilt Axum runtime smoke passed the exercised +fail-open and private-response exclusion scenarios. No high or medium finding +remains. Opus retains the live-signing evidence gap and accepted two-pass +boundary/metadata separation as low findings. + +Publication hold: no pushes, remote PR changes, comments, email or deployments. +All review corrections remain local; no landing requested. + +Independent pre-score assessments: `openai-think` (GPT-6 Astra) and +`anthropic-think` (Claude Fable 5) completed and identified the initial corrections. +The local finalization corrections now cover raw-markup boundary selection, +entity decoding, output expansion, negative-cache overwrite races, invalid +cache recovery, adapter receive-time response limits and credential-bound +HTTP cache bypass. The matching server contract also confirms that equal +canonical text can reuse the first URL-bearing manifest across URL or title +changes. Final pre-score review additionally corrected raw-text scan complexity, +full-article insertion-offset allocation and canonical ignored custom-element +prefix handling. Full local gates `1792869014289` and `1792870296283` passed all +38 commands. Cycle 2 additionally removes full-fragment materialization, +repeated metadata scans and the second rendered-document allocation. Its +93 module tests and module clippy pass, as does the final 38-command full gate. + +## Overview + +Encypher's text provenance contribution signed article pages from inside core, +through an HTML stream processor factory, a request context type and core +changes of its own. The stack in #777 moves every vendor out of core and lets a +module change a page only through a middleware. This work moves the +contribution onto those interfaces as `crates/provenance/encypher` +(`provenance.encypher`) and adds the neutral core seams it needs. Design: +`docs/superpowers/specs/2026-10-24-encypher-provenance-module-design.md`. + +## Objectives + +1. Port every existing Encypher behavior (gates, policy and plan caches, sign + call, render, fail-open, logging) to a vendor-owned module crate. +2. Add only vendor-neutral core seams: an awaited asynchronous finish for + middleware stream processors, an optional platform timer, response facts + for middleware, runtime KV on the adapters that lacked it. +3. Remove obsolete paths in the port: the `enabled` flag, `path_prefixes`, + `[integrations.encypher]`, the stream processor factory, the shared-template + refusal and the synchronous publisher body path. +4. Keep the permission contract: no permission required, no gated value read, + no page the origin marks private sent. +5. Publish a reviewable pull request against the stack head without changing + any upstream branch. + +## Work Breakdown + +Implementation (completed; evidence historical, see Completion Notes): + +1. [x] Inspect epic #777, PR #1423 head and the existing Encypher branches +2. [x] Create an isolated worktree and branch from 5be114773 +3. [x] `PlatformTimer` on `RuntimeServices`, with neutral documentation +4. [x] Fastly opens the KV store linked as `trusted_server_kv` +5. [x] `StreamProcessor::async_finish` and `finish`, composed in order by + every composite processor +6. [x] Every awaiting publisher path ends its processor with `finish`; the + synchronous `stream_publisher_body` path removed +7. [x] `MiddlewareContext::status` and `origin_private`, read before core's + cache policy +8. [x] Axum runtime KV from `TRUSTED_SERVER_KV_PATH`, on the blocking pool +9. [x] Timers on Cloudflare, Spin and Axum +10. [x] `crates/provenance/encypher`: config, secret setting, middleware, + ported processor and tests, publisher-path tests +11. [x] Module offered on all four adapters and the CLI +12. [x] Auction-hold regression test with an asynchronously finished document +13. [x] Documentation: configuration guide, Axum guide, example config, design +14. [x] Data-handling wording narrowed to what the gates prove +15. [x] Repository CI gates run locally once (historical; some adapter aliases + ran before the later rustfmt-only change and the auction-hold test) +16. [x] Axum runtime smoke with a controlled origin (fail-open and + origin-private paths only; the live signing endpoint was unavailable, so + live signing was not exercised) +17. [x] Draft pull request opened and its body updated with evidence + +Review loop, Phase 5 (complete; Gate B cleared at 9.5): + +18. [x] PRD finalization for the review loop (historical evidence retained; + premature completion removed; local Prettier check passed) +19. [x] Concise documentation sweep; timeout, retention and Axum persistence + wording corrected, with module deployment release notes +20. [x] Simplification sweep (behavior-preserving, covering tests named) +21. [x] Security sweep: entry-to-sink, fail-open, private and tenant cache + boundaries, finish ordering and cancellation, timers and budgets, + credential treatment, config and runtime registration, cache integrity +22. [x] Fresh verification: final full gate `1792871494395`, all 38 commands + passed, including 93 module tests and 3,022 core tests. Rebuilt Axum + runtime smoke verified fail-open and private/Set-Cookie exclusion +23. [x] Completion gate (Gate B) cleared at bar 9.5 by both reviewers on + correctness, simplification and security; cycle 2 scores above + +## Success Criteria + +| Criterion | Result | +| ----------------------------------------- | -------------------------------------------------------------------------------------------------- | +| Signing works through the publisher path | Met in process: buffered and streaming finalizers sign against a stub origin and stub API | +| Every failure passes the page through | Signed-output expansion regression passes for fresh and cached plans; original held bytes retained | +| A private origin page is never sent | Met: in-process test and runtime smoke `gate=origin_private` | +| Core platform-name neutrality lint passes | Met: `scripts/lint-core-platform-names.py` exit 0 | +| CI gates green | Final full run `1792871494395`: all 38 commands passed | +| Live API signing | Not met: not exercised, see limits | + +The internal completion gate cleared. Live signing remains a separate +deployment prerequisite rather than an inferred result of these checks. +In-process signing against the stub API does not prove interoperability. +The matching policy-digest implementation was located in the original commercial +`trusted-server-provenance` worktree at `bfe8f050` (policy change `832ded4b`). +The deployed API revision and current signing admission remain unverified. + +## Completion Notes + +Tests were ported from the earlier Encypher branches and written alongside the +port. The work was not strictly test-first, and no baseline run of the stack +was recorded before the change. + +The command runs below are historical, from the implementation work. Fresh +review evidence is recorded separately under Current review verification. + +Commands run in the worktree, with exit codes: + +- `cargo fmt --all -- --check`: 0 +- `cargo clippy-fastly`, `clippy-axum`, `clippy-cloudflare`, + `clippy-cloudflare-wasm`, `clippy-spin-native`, `clippy-spin-wasm`, + `clippy-cli`, `clippy-codegen`: 0 each +- `cargo test -p trusted-server-provenance-encypher`: 65 passed +- `cargo test -p trusted-server-core --lib`: 3020 passed; the auction-hold + regression test added after it: 1 passed +- `cargo test-fastly`, `test-axum`, `test-cloudflare`, `test-spin`: no + failures (run before the rustfmt-only change and the auction-hold test) +- `cargo check-cloudflare`, `check-spin`, `build-fastly`: no errors +- `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity`: 0 +- `scripts/lint-wasm-clock.sh`: 0 +- `python3 scripts/lint-core-platform-names.py`: 0 +- `cd crates/trusted-server-js/lib && npx vitest run`: 0; `npm run format`: 0 +- `cd docs && npm run format`: 0; prettier check of markdown outside `docs/`: 0 +- `cargo test -p trusted-server-core --test template_build_digest`: 0; its + clippy with `-D warnings`: 0 +- docs-proposal: `uvx --from shellcheck-py shellcheck scripts/docs-proposal/*.sh` + (shellcheck is not installed on the host): 0; + `node --test scripts/docs-proposal/hunks.test.mjs`: 0; + `scripts/docs-proposal/test.sh`: 0 + +Runtime smoke, Axum dev server, throwaway script and mock origin, no real +credentials. The module was selected with a `[[serve]]` entry on `/news/`, +runtime KV on, and `[geo] assume_single_jurisdiction = true`, which the stack +requires once `[ec] module` is set. Observed: + +```text +[edgezero] config validate (typed): edgezero.toml + trusted-server.toml OK (strict) +page=public status=200 harbor_text=1 marker=0 +page=private status=200 harbor_text=1 marker=0 +INFO [trusted_server_provenance_encypher] encypher: registered for domain=publisher.example.com +INFO [trusted_server_provenance_encypher::processor] encypher: outcome=passed_through reason=http_error +DEBUG [trusted_server_provenance_encypher] encypher: document not held, gate=origin_private +``` + +`scripts/smoke-axum.sh` itself does not set the `[geo]` value and stops at +that check. This branch leaves `scripts/` and the core settings, config and +Edge Cookie code unchanged from 5be114773; the script was not run on the base +commit. + +Limits: + +- Live API: the runtime smoke pointed the module at a signing endpoint that + was not available, to exercise fail-open. Successful signing was tested in + process only, against a stub API. The live Encypher API was not exercised. +- Personalized pages: the module skips pages whose origin set a cookie or + forbade shared caching, and cannot detect others. A deployment places it on + public article routes. +- Spin's declared KV store also holds the settings. The module's four + positive and negative namespaces are versioned, but Spin has no item TTL. + Expired entries fail closed on read and are deleted best effort; abandoned + content keys require the namespace-retirement procedure in the Spin guide. +- Fastly has no guest timer. Its backend timeout bounds the first byte and + between-byte gaps, not total elapsed time, so `max_hold_ms` is not a total + deadline there. A trickling signing peer can exceed it. +- Equal canonical article text intentionally shares an Encypher plan across + page URL or title changes. The manifest can retain the first canonical URL, + as allowed by the matching server contract. + +### Current review verification + +The unattended gate is +`bun /home/agentdesk/scratch/team554-review-ci.mjs full`. +Run `1792867884379` passed 37 of 38 commands, including module/core and all +adapter tests, native/WASM builds and checks, CLI/parity, JavaScript, docs and +proposal checks. `clippy-spin-wasm` rejected an obsolete response-policy +wrapper; it was removed and its tests migrated to the existing bounded helper. +Subsequent runs `1792869014289` and `1792870296283` passed all 38 commands. +Final run `1792871494395` passed all 38 commands, including 93 module tests, +3,022 core tests, every adapter test/build/lint gate and documentation checks. +Shellcheck also passed separately. A superseded in-progress run was cancelled +before the final offset-vector preallocation correction; it is not a pass. + +Review regressions reproduced canonical custom-element text divergence. +The raw-text decoy regression passed functionally but took 5.78 seconds for +10,000 unrelated closing tags before correction. Sparse embedding over +4.9 million visible article characters passed functionally but used a +document-sized offset vector; observed process max RSS was 69,693,440 bytes. +The corrections filter unrelated raw end names before scanning attributes and +stop offset decoding at the highest signed index. No fixtures were changed. + +Rebuilt Axum smoke observed public, private and Set-Cookie responses with +status 200 and original article text preserved. Private and Set-Cookie paths +made no API connection. A stalled TLS handshake failed open with `timeout` +in 0.365 seconds on the final rebuilt binary. The scoped local CA was rejected despite passing +the Python trust control; successful signing, query omission and HTTP 503 +handling were not exercised by this runtime smoke. No trust checks were +weakened. Ports and temporary certificate keys were cleaned. + +Cycle 2 replaces whole-article fragment vectors and post-allocation text +truncation with a stoppable shared scanner and canonical visitor. Extraction +stops at 50,000 scalars; title decoding stops at 500 and occurs only on a sign +miss. One metadata pass saves manifest presence, title range and head-close +offset. Rendering merges markers and escaped manifest metadata in one +exact-capacity output allocation. Equal-offset markers precede metadata; +characterization tests preserve malformed-head, absent-head and empty-URL +behavior. Fixtures remain unchanged. + +Final evidence is retained under +`/home/agentdesk/scratch/team554-review-evidence/`, with final CI rows and +per-command logs keyed by `1792871494395` and runtime results under `runtime/`. +The security audit is under +`/home/agentdesk/security-audits/trusted-server/team554-review/`. +No source or fixture changes followed the final gate. This acceptance record +was finalized and archived locally afterward; no publication was authorized. +Historical test-first and plan-gate deviations remain recorded above. +The earlier outbound email attempt remains uncertain and must not be blindly +retried; see the TEAM_554 company session record. diff --git a/crates/middleware/51degrees/src/lib.rs b/crates/middleware/51degrees/src/lib.rs index 4b5a557ef8..39ff781681 100644 --- a/crates/middleware/51degrees/src/lib.rs +++ b/crates/middleware/51degrees/src/lib.rs @@ -658,6 +658,9 @@ mod tests { request_host: "www.example.com", request_scheme: "https", origin_host: "origin.example.com", + status: http::StatusCode::OK, + origin_private: false, + response_headers: None, platform: test_support::platform(), document_state: state, max_buffered_script_bytes: 1 << 20, diff --git a/crates/provenance/encypher/Cargo.toml b/crates/provenance/encypher/Cargo.toml new file mode 100644 index 0000000000..4bf0743835 --- /dev/null +++ b/crates/provenance/encypher/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "trusted-server-provenance-encypher" +description = "Encypher provenance: signs the article on each page a [[serve]] entry names it for, with invisible markers and a C2PA manifest link, and the exact bytes of its media, delivering every item it cannot sign as it was and reporting it unsigned." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +base64 = { workspace = true } +bytes = { workspace = true } +ed25519-dalek = { workspace = true } +edgezero-core = { workspace = true } +ego-tree = { workspace = true } +error-stack = { workspace = true } +futures = { workspace = true } +html5ever = { workspace = true } +http = { workspace = true } +log = { workspace = true } +roxmltree = { workspace = true } +scraper = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } +validator = { workspace = true } +wasmi = { workspace = true } +web-time = { workspace = true } + +[dev-dependencies] +async-trait = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } +wat = { workspace = true } + +# The visible owner of this vendor crate. +[package.metadata.maintainers] +owner = "Encypher" +status = "vendor owned" diff --git a/crates/provenance/encypher/src/api.rs b/crates/provenance/encypher/src/api.rs new file mode 100644 index 0000000000..ddf89b92a7 --- /dev/null +++ b/crates/provenance/encypher/src/api.rs @@ -0,0 +1,303 @@ +use serde::{Deserialize, Serialize}; + +/// A 2 MiB cap accommodates roughly 600 KiB surrogate-escaped plans at the 50,000-scalar input limit. +pub const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; +/// Maximum visible-text length sent for signing, measured in Unicode scalars. +pub const MAX_TEXT_SCALARS: usize = 50_000; +/// Policy responses contain only one digest; cap them independently of plans. +pub const MAX_POLICY_RESPONSE_BYTES: usize = 1024; + +/// JSON request body for the Encypher CDN signing endpoint. +#[derive(Serialize)] +pub struct SignRequest<'a> { + /// Canonical visible article text. + pub text: &'a str, + /// Absolute page URL without its query string. + pub page_url: &'a str, + /// Optional page title capped at 500 Unicode scalars. + #[serde(skip_serializing_if = "Option::is_none")] + pub document_title: Option<&'a str>, + /// Boundary detection label used for the article. + pub boundary_selector: &'a str, + /// Server policy used to select this article's cache namespace. + pub expected_policy_digest: &'a str, +} + +/// One invisible-marker insertion returned by the signing endpoint. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PlanOperation { + /// Visible codepoint index after which to insert, or -1 for before the first. + pub insert_after_index: i64, + /// Invisible marker characters to insert. + pub marker: String, +} + +/// A complete marker embedding plan. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EmbeddingPlan { + /// Index unit, which must be `codepoint`. + pub index_unit: String, + /// Ordered marker insertion operations. + pub operations: Vec, +} + +/// Parsed response from the Encypher CDN signing endpoint. +#[derive(Debug, Deserialize)] +pub struct SignResponse { + /// Whether signing succeeded. + pub success: bool, + /// Marker plan when signing succeeded. + #[serde(default)] + pub embedding_plan: Option, + /// C2PA manifest URL when signing succeeded. + #[serde(default)] + pub manifest_url: Option, + /// Stable failure reason when signing did not succeed. + #[serde(default)] + pub error: Option, + /// Actual policy used by the server; required before a successful plan is used. + #[serde(default)] + pub policy_digest: Option, +} + +/// Fail-closed signing-response parse errors. +#[derive(Debug, PartialEq)] +pub enum ApiError { + /// The response exceeded [`MAX_RESPONSE_BYTES`]. + TooLarge, + /// The response was not valid JSON for [`SignResponse`]. + Malformed, +} + +/// Parses a bounded signing response. +/// +/// # Errors +/// +/// Returns [`ApiError::TooLarge`] above the response limit and +/// [`ApiError::Malformed`] when JSON decoding fails. +pub fn parse_sign_response(body: &[u8]) -> Result { + if body.len() > MAX_RESPONSE_BYTES { + return Err(ApiError::TooLarge); + } + serde_json::from_slice(body).map_err(|_| ApiError::Malformed) +} + +/// A policy digest is exactly a lowercase SHA-256 hex value. +pub fn valid_policy_digest(digest: &str) -> bool { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Parse the policy endpoint's strict, bounded response. +/// +/// # Errors +/// +/// Rejects oversized JSON, extra or duplicate fields, and invalid digests. +pub fn parse_policy_response(body: &[u8]) -> Result { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct PolicyResponse { + policy_digest: String, + } + if body.len() > MAX_POLICY_RESPONSE_BYTES { + return Err(ApiError::TooLarge); + } + let response: PolicyResponse = serde_json::from_slice(body).map_err(|_| ApiError::Malformed)?; + if !valid_policy_digest(&response.policy_digest) { + return Err(ApiError::Malformed); + } + Ok(response.policy_digest) +} + +/// Reasons an embedding plan is unsafe or cannot address the visible text. +#[derive(Debug, PartialEq)] +pub enum PlanError { + /// The plan does not use Unicode codepoint indices. + IndexUnit, + /// An insertion index is outside the visible-text range. + IndexOutOfRange, + /// A marker is empty. + EmptyMarker, + /// A marker contains a character outside the edge-core invisible set. + DisallowedMarkerChar, + /// The plan has no operations. + Empty, + /// The manifest URL is present but not an absolute `https` URL. + ManifestUrl, +} + +/// Validates a signing result before use: index bounds and the invisible marker +/// alphabet for every operation, and a manifest URL that is absent or `https`. +/// +/// This is the only check between the API or KV and the page, so callers must +/// pass the scalar count of the exact text that was signed. +/// +/// # Errors +/// +/// Returns a [`PlanError`] for the first invalid property. +pub fn validate_plan( + plan: &EmbeddingPlan, + manifest_url: Option<&str>, + visible_scalars: usize, +) -> Result<(), PlanError> { + if plan.index_unit != "codepoint" { + return Err(PlanError::IndexUnit); + } + if plan.operations.is_empty() { + return Err(PlanError::Empty); + } + for operation in &plan.operations { + if operation.insert_after_index < -1 { + return Err(PlanError::IndexOutOfRange); + } + if operation.insert_after_index >= 0 { + let Ok(index) = usize::try_from(operation.insert_after_index) else { + return Err(PlanError::IndexOutOfRange); + }; + if index >= visible_scalars { + return Err(PlanError::IndexOutOfRange); + } + } + if operation.marker.is_empty() { + return Err(PlanError::EmptyMarker); + } + if !operation.marker.chars().all(is_allowed_marker_char) { + return Err(PlanError::DisallowedMarkerChar); + } + } + if let Some(url) = manifest_url { + let https = url::Url::parse(url).is_ok_and(|url| { + url.scheme() == "https" && url.host_str().is_some_and(|h| !h.is_empty()) + }); + if !https { + return Err(PlanError::ManifestUrl); + } + } + Ok(()) +} + +fn is_allowed_marker_char(ch: char) -> bool { + matches!(ch, '\u{fe00}'..='\u{fe0f}' | '\u{e0100}'..='\u{e01ef}' | '\u{feff}') +} + +#[cfg(test)] +mod tests { + use super::*; + + fn single_op_plan() -> EmbeddingPlan { + EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_string(), + }], + } + } + + #[test] + fn policy_response_requires_only_a_lowercase_sha256_digest() { + let digest = "ab".repeat(32); + let body = format!(r#"{{"policy_digest":"{digest}"}}"#); + assert_eq!(parse_policy_response(body.as_bytes()), Ok(digest.clone())); + for body in [ + "{}".to_owned(), + r#"{"policy_digest":null}"#.to_owned(), + r#"{"policy_digest":42}"#.to_owned(), + format!(r#"{{"policy_digest":"{}"}}"#, digest.to_uppercase()), + format!(r#"{{"policy_digest":"{}"}}"#, "g".repeat(64)), + format!(r#"{{"policy_digest":"{}"}}"#, "a".repeat(63)), + format!(r#"{{"policy_digest":"{digest}","extra":true}}"#), + format!(r#"{{"policy_digest":"{digest}","policy_digest":"{digest}"}}"#), + format!(r#"{{"policy_digest":"{digest}"}} trailing"#), + ] { + assert_eq!( + parse_policy_response(body.as_bytes()), + Err(ApiError::Malformed) + ); + } + assert_eq!( + parse_policy_response(&vec![b' '; MAX_POLICY_RESPONSE_BYTES + 1]), + Err(ApiError::TooLarge) + ); + } + + #[test] + fn parses_and_validates_a_response_plan() { + let body = r#"{"success":true,"embedding_plan":{"index_unit":"codepoint","operations":[{"insert_after_index":0,"marker":"︀"}]},"manifest_url":"https://manifest.example.com/x"}"#; + let response = parse_sign_response(body.as_bytes()).expect("should parse"); + let plan = response.embedding_plan.expect("should contain plan"); + assert_eq!( + validate_plan(&plan, response.manifest_url.as_deref(), 1), + Ok(()) + ); + } + + #[test] + fn manifest_url_must_be_absent_or_https() { + let plan = single_op_plan(); + assert_eq!(validate_plan(&plan, None, 1), Ok(())); + for url in [ + "", + "http://manifest.example.com/x", + "javascript:alert(1)", + "data:text/html,", + "/api/v1/public/cdn/manifest/1", + "https://", + ] { + assert_eq!( + validate_plan(&plan, Some(url), 1), + Err(PlanError::ManifestUrl), + "{url} should be rejected" + ); + } + } + + #[test] + fn rejects_markup_and_bad_ranges() { + let markup = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "".to_string(), + }], + }; + assert_eq!( + validate_plan(&markup, None, 1), + Err(PlanError::DisallowedMarkerChar) + ); + assert_eq!( + validate_plan(&single_op_plan(), None, 0), + Err(PlanError::IndexOutOfRange) + ); + } + + #[test] + fn parses_realistic_large_marker_response() { + let escaped_marker = r"\uDB40\uDD00".repeat(50_000); + let body = format!( + r#"{{"success":true,"embedding_plan":{{"index_unit":"codepoint","operations":[{{"insert_after_index":0,"marker":"{escaped_marker}"}}]}}}}"# + ); + assert!(body.len() > 600_000); + let response = + parse_sign_response(body.as_bytes()).expect("large valid response should parse"); + let plan = response + .embedding_plan + .expect("response should contain a plan"); + assert_eq!(plan.operations[0].marker.chars().count(), 50_000); + } + + #[test] + fn enforces_size_and_empty_plan_limits() { + assert_eq!( + parse_sign_response(&vec![b' '; MAX_RESPONSE_BYTES + 1]).map(|_| ()), + Err(ApiError::TooLarge) + ); + let empty = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: Vec::new(), + }; + assert_eq!(validate_plan(&empty, None, 1), Err(PlanError::Empty)); + } +} diff --git a/crates/provenance/encypher/src/boundary.rs b/crates/provenance/encypher/src/boundary.rs new file mode 100644 index 0000000000..e4aa722ece --- /dev/null +++ b/crates/provenance/encypher/src/boundary.rs @@ -0,0 +1,464 @@ +use super::text::{StructuralTag, TagKind, scan_structural_tags}; + +const CMS_CONTENT_CLASSES: [&str; 13] = [ + "entry-content", + "wp-block-post-content", + "post-content", + "article-body", + "article-content", + "story-body", + "story-body-text", + "gh-content", + "w-richtext", + "sqs-block-content", + "body markup", + "e-content", + "RichTextStoryBody", +]; + +/// The UTF-8 byte range and detection label of an article's outer HTML. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Boundary { + /// Inclusive byte offset of the opening element. + pub start: usize, + /// Exclusive byte offset after the first matching closing element. + pub end: usize, + /// Detection-chain selector label. + pub selector: String, +} + +#[derive(Debug, Clone, Copy)] +struct ElementMatch { + start: usize, + end: usize, +} + +#[derive(Clone, Copy)] +enum FirstMatch<'a> { + Unseen, + Open { start: usize, name: &'a str }, + Complete(ElementMatch), +} + +impl<'a> FirstMatch<'a> { + fn observe(&mut self, tag: StructuralTag<'a>, opening_matches: bool) { + match (*self, tag.kind) { + (Self::Unseen, TagKind::Opening) if opening_matches => { + *self = Self::Open { + start: tag.start, + name: tag.name, + }; + } + (Self::Open { start, name }, TagKind::Closing) + if tag.name.eq_ignore_ascii_case(name) => + { + *self = Self::Complete(ElementMatch { + start, + end: tag.end, + }); + } + _ => {} + } + } + + fn complete(self) -> Option { + match self { + Self::Complete(element) => Some(element), + Self::Unseen | Self::Open { .. } => None, + } + } +} + +#[derive(Clone, Copy)] +struct ActiveCount { + start: usize, + paragraphs: usize, +} + +#[derive(Clone, Copy)] +struct ScoredMatch { + element: ElementMatch, + paragraphs: usize, +} + +#[derive(Clone, Copy)] +struct CountedMatches { + name: &'static str, + active: Option, + best: Option, +} + +impl CountedMatches { + const fn new(name: &'static str) -> Self { + Self { + name, + active: None, + best: None, + } + } + + fn observe(&mut self, tag: StructuralTag<'_>) { + match tag.kind { + TagKind::Opening => { + if let Some(active) = &mut self.active { + if tag.name.eq_ignore_ascii_case("p") { + active.paragraphs += 1; + } + } else if tag.name.eq_ignore_ascii_case(self.name) { + self.active = Some(ActiveCount { + start: tag.start, + paragraphs: 0, + }); + } + } + TagKind::Closing if tag.name.eq_ignore_ascii_case(self.name) => { + let Some(active) = self.active.take() else { + return; + }; + let candidate = ScoredMatch { + element: ElementMatch { + start: active.start, + end: tag.end, + }, + paragraphs: active.paragraphs, + }; + if self + .best + .is_none_or(|best| candidate.paragraphs > best.paragraphs) + { + self.best = Some(candidate); + } + } + TagKind::Closing => {} + } + } +} + +enum Selector<'a> { + Tag(&'a str), + Class(&'a str), + Attribute(&'a str, &'a str), +} + +impl Selector<'_> { + fn matches(&self, tag: &StructuralTag<'_>) -> bool { + match *self { + Self::Tag(name) => tag.name.eq_ignore_ascii_case(name), + Self::Class(name) => tag + .attribute("class") + .is_some_and(|value| class_matches(value, name)), + Self::Attribute(attribute, value) => attribute_matches(tag, attribute, value), + } + } +} + +/// Finds an article boundary using the edge-core priority chain, excluding JSON-LD P5. +#[must_use] +pub fn find_article_boundary(html: &str, override_selector: Option<&str>) -> Option { + let override_label = override_selector.filter(|selector| !selector.is_empty()); + let override_matcher = override_label.and_then(parse_selector); + let mut override_element = FirstMatch::Unseen; + let mut article = CountedMatches::new("article"); + let mut cms = [FirstMatch::Unseen; CMS_CONTENT_CLASSES.len()]; + let mut itemprop = FirstMatch::Unseen; + let mut main = FirstMatch::Unseen; + let mut role = FirstMatch::Unseen; + let mut body = FirstMatch::Unseen; + let mut div_cluster = CountedMatches::new("div"); + let mut section_cluster = CountedMatches::new("section"); + let mut main_cluster = CountedMatches::new("main"); + + // All priority candidates share one structural pass. Each first-match + // state deliberately keeps an unclosed first opening from being rescued + // by a later one, matching the previous selector scans. + scan_structural_tags(html, |tag| { + let opening = tag.kind == TagKind::Opening; + override_element.observe( + tag, + opening + && override_matcher + .as_ref() + .is_some_and(|selector| selector.matches(&tag)), + ); + let class = opening.then(|| tag.attribute("class")).flatten(); + for (state, expected) in cms.iter_mut().zip(CMS_CONTENT_CLASSES) { + state.observe( + tag, + class.is_some_and(|value| class_matches(value, expected)), + ); + } + itemprop.observe( + tag, + opening && attribute_matches(&tag, "itemprop", "articleBody"), + ); + main.observe(tag, opening && tag.name.eq_ignore_ascii_case("main")); + role.observe(tag, opening && attribute_matches(&tag, "role", "main")); + body.observe(tag, opening && tag.name.eq_ignore_ascii_case("body")); + article.observe(tag); + div_cluster.observe(tag); + section_cluster.observe(tag); + main_cluster.observe(tag); + true + }); + + if let (Some(selector), Some(element)) = (override_label, override_element.complete()) { + return Some(boundary(element, format!("override:{selector}"))); + } + if let Some(selected) = article.best { + return Some(boundary(selected.element, "article")); + } + for (state, class_name) in cms.into_iter().zip(CMS_CONTENT_CLASSES) { + if let Some(element) = state.complete() { + return Some(boundary(element, format!("class:{class_name}"))); + } + } + if let Some(element) = itemprop.complete() { + return Some(boundary(element, "itemprop:articleBody")); + } + if let Some(element) = main.complete() { + return Some(boundary(element, "main")); + } + if let Some(element) = role.complete() { + return Some(boundary(element, "role:main")); + } + + let mut cluster = None; + for candidate in [div_cluster.best, section_cluster.best, main_cluster.best] + .into_iter() + .flatten() + .filter(|candidate| candidate.paragraphs > 4) + { + if cluster.is_none_or(|best: ScoredMatch| candidate.paragraphs > best.paragraphs) { + cluster = Some(candidate); + } + } + if let Some(selected) = cluster { + return Some(boundary(selected.element, "p-cluster")); + } + body.complete().map(|element| boundary(element, "body")) +} + +fn boundary(element: ElementMatch, selector: impl Into) -> Boundary { + Boundary { + start: element.start, + end: element.end, + selector: selector.into(), + } +} + +/// Checks the same restricted selector grammar used for article matching. +pub(super) fn is_valid_selector(selector: &str) -> bool { + parse_selector(selector).is_some() +} + +fn parse_selector(selector: &str) -> Option> { + if let Some(class_name) = selector.strip_prefix('.') { + return is_class_identifier(class_name).then_some(Selector::Class(class_name)); + } + if selector.starts_with('[') { + let (attribute, value) = parse_attribute_selector(selector)?; + return Some(Selector::Attribute(attribute, value)); + } + (selector + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphabetic) + && selector + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-'))) + .then_some(Selector::Tag(selector)) +} + +fn is_identifier_character(character: char) -> bool { + character.is_ascii_alphanumeric() || matches!(character, '_' | '-') || character >= '\u{80}' +} + +fn is_class_identifier(name: &str) -> bool { + let mut characters = name.chars(); + let Some(mut first) = characters.next() else { + return false; + }; + if first == '-' { + let Some(next) = characters.next() else { + return false; + }; + first = next; + } + (first.is_ascii_alphabetic() || matches!(first, '_' | '-') || first >= '\u{80}') + && characters.all(is_identifier_character) +} + +fn parse_attribute_selector(selector: &str) -> Option<(&str, &str)> { + let content = selector.strip_prefix('[')?.strip_suffix(']')?; + let (attribute, raw_value) = content.split_once('=')?; + if attribute.is_empty() + || !attribute + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + return None; + } + let value = match ( + raw_value.as_bytes().first().copied(), + raw_value.as_bytes().last().copied(), + ) { + (Some(first @ (b'"' | b'\'')), Some(last)) if first == last => { + let value = raw_value.get(1..raw_value.len().saturating_sub(1))?; + return (!value.is_empty() + && !value.chars().any(|character| { + character.is_control() || matches!(character, '"' | '\'' | ']' | '\\') + })) + .then_some((attribute, value)); + } + (Some(b'"' | b'\''), _) => return None, + _ => raw_value, + }; + (!value.is_empty() && value.chars().all(is_identifier_character)).then_some((attribute, value)) +} + +fn attribute_matches(tag: &StructuralTag<'_>, attribute: &str, expected: &str) -> bool { + tag.attribute(attribute) + .is_some_and(|value| value.eq_ignore_ascii_case(expected)) +} + +fn class_matches(value: &str, expected: &str) -> bool { + let mut expected_parts = expected.split_ascii_whitespace(); + let Some(first) = expected_parts.next() else { + return false; + }; + let second = expected_parts.next(); + if expected_parts.next().is_some() { + return false; + } + let mut previous_was_first = false; + for class in value.split_ascii_whitespace() { + if second.is_some_and(|second| previous_was_first && class.eq_ignore_ascii_case(second)) { + return true; + } + previous_was_first = class.eq_ignore_ascii_case(first); + if second.is_none() && previous_was_first { + return true; + } + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn follows_priority_chain_and_byte_offsets() { + let html = "🐟

Main.

Chosen.

"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!(boundary.selector, "article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("

Chosen.

") + ); + } + + #[test] + fn nested_same_name_uses_first_closing_tag() { + let html = "
outer
inner
tail
"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("
outer
inner
") + ); + } + + #[test] + fn closing_tags_match_ascii_case_insensitively() { + let html = "

One.

x
"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("

One.

") + ); + let boundary = + find_article_boundary(html, Some(".entry-content")).expect("should find override"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("
x
") + ); + } + + #[test] + fn article_detection_ignores_inert_opening_markup() { + let real = "

Real article.

"; + for inert in [ + "", + r#""#, + r#"
"#, + ] { + let html = format!("{inert}{real}"); + let boundary = find_article_boundary(&html, None).expect("should find real article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some(real), + "inert markup must not become the article: {inert}" + ); + } + } + + #[test] + fn inert_closing_markup_does_not_truncate_an_article() { + for inert in [ + "", + r#""#, + r#"Attribute text."#, + ] { + let html = format!("

Start.

{inert}

End.

"); + let boundary = find_article_boundary(&html, None).expect("should find article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some(html.as_str()), + "inert markup must not close the article: {inert}" + ); + } + } + + #[test] + fn invalid_less_than_and_declarations_do_not_hide_or_create_articles() { + let real = "

Real article.

"; + for prefix in [ + "text < y ", + "< article>

False spaced article.

", + "

False declaration.

", + "

False instruction.

", + " remainder\">", + ] { + let html = format!("{prefix}{real}"); + let boundary = find_article_boundary(&html, None).expect("should find real article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some(real), + "invalid markup must not hide or create the article: {prefix}" + ); + } + + let html = "

Start.

End.

"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!(html.get(boundary.start..boundary.end), Some(html)); + } + + #[test] + fn dense_documents_select_articles_without_a_document_sized_tag_index() { + let padding = "".repeat(700_000); + let first = "

First.

"; + let second = "

Second.

Wins.

"; + let html = format!("{padding}{first}{second}"); + + let boundary = find_article_boundary(&html, None).expect("should find article"); + assert_eq!(boundary.selector, "article"); + assert_eq!(html.get(boundary.start..boundary.end), Some(second)); + } + + #[test] + fn selects_largest_paragraph_cluster_at_five() { + let html = "

1

2

3

4

5

"; + let boundary = find_article_boundary(html, None).expect("should find cluster"); + assert_eq!(boundary.selector, "p-cluster"); + } +} diff --git a/crates/provenance/encypher/src/data_uri.rs b/crates/provenance/encypher/src/data_uri.rs new file mode 100644 index 0000000000..8edf13e054 --- /dev/null +++ b/crates/provenance/encypher/src/data_uri.rs @@ -0,0 +1,314 @@ +//! `data:` URIs carrying media: parsing, exact decoding and lossless +//! re-encoding in the URI's own encoding. +//! +//! A URI is rewritten as `data:` (as written), its header text between the +//! scheme and the comma (as written), and the signed bytes encoded the way +//! the original payload was: base64, padded only if the original was, or +//! percent-encoding with a set that is safe in an HTML attribute (quoted or +//! not), a `srcset` candidate and a CSS `url()`. Decoding the new URI gives +//! back exactly the signed bytes. + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use base64::engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig}; + +use crate::media_api::canonical_media_type; + +/// Forgiving base64, as browsers decode a `data:` URI: padding optional and +/// trailing bits ignored. ASCII whitespace is removed before decoding. +const FORGIVING: GeneralPurpose = GeneralPurpose::new( + &base64::alphabet::STANDARD, + GeneralPurposeConfig::new() + .with_decode_padding_mode(DecodePaddingMode::Indifferent) + .with_decode_allow_trailing_bits(true), +); + +/// One `data:` URI whose media type is media the module signs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DataUri<'a> { + /// The scheme as written, `data:` in any case. + scheme: &'a str, + /// Everything between the scheme and the comma, as written. + header: &'a str, + /// The payload after the comma. + payload: &'a str, + base64: bool, +} + +/// Why a media `data:` URI cannot be decoded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Malformed; + +/// Whether `media_type` is media: an image, audio, video or font. +pub(crate) fn is_media_type(media_type: &str) -> bool { + ["image/", "audio/", "video/", "font/"] + .iter() + .any(|prefix| media_type.starts_with(prefix) && media_type.len() > prefix.len()) +} + +impl<'a> DataUri<'a> { + /// The `data:` URI `value` is, when it declares media. Any other value, + /// or a `data:` URI of another type, is `None`. + pub(crate) fn parse(value: &'a str) -> Option { + let scheme = value.get(..5)?; + if !scheme.eq_ignore_ascii_case("data:") { + return None; + } + let rest = value.get(5..)?; + let comma = rest.find(',')?; + let header = rest.get(..comma)?; + let payload = rest.get(comma + 1..)?; + let uri = Self { + scheme, + header, + payload, + base64: header + .rsplit(';') + .next() + .is_some_and(|last| last.trim().eq_ignore_ascii_case("base64")) + && header.contains(';'), + }; + is_media_type(&uri.media_type()).then_some(uri) + } + + /// The bare, lower-case media type the URI declares. + pub(crate) fn media_type(&self) -> String { + canonical_media_type(self.header.split(';').next().unwrap_or_default().trim()) + } + + /// The declared type with its parameters, as written, without `;base64`, + /// sent as the source's type hint. + pub(crate) fn declared_type(&self) -> &'a str { + if self.base64 { + self.header + .rsplit_once(';') + .map_or(self.header, |(declared, _)| declared) + } else { + self.header + } + } + + /// The exact bytes the URI carries. + /// + /// # Errors + /// + /// Returns [`Malformed`] for base64 a browser would refuse. + pub(crate) fn decode(&self) -> Result, Malformed> { + if self.base64 { + let compact: Vec = self + .payload + .bytes() + .filter(|byte| !byte.is_ascii_whitespace()) + .collect(); + return FORGIVING.decode(compact).map_err(|_| Malformed); + } + Ok(percent_decode(self.payload.as_bytes())) + } + + /// The URI for `bytes`, with this URI's scheme and header text, encoded + /// as this URI's payload was. + pub(crate) fn encode(&self, bytes: &[u8]) -> String { + let mut out = String::with_capacity( + self.scheme.len() + self.header.len() + 1 + bytes.len().saturating_mul(3), + ); + out.push_str(self.scheme); + out.push_str(self.header); + out.push(','); + if self.base64 { + let encoded = STANDARD.encode(bytes); + if self.payload.trim_end().ends_with('=') || self.unpadded_length_is_aligned() { + out.push_str(&encoded); + } else { + out.push_str(encoded.trim_end_matches('=')); + } + } else { + percent_encode_into(&mut out, bytes); + } + out + } + + /// Whether the original payload needed no padding, so writing padding + /// keeps its form. + fn unpadded_length_is_aligned(&self) -> bool { + self.payload + .bytes() + .filter(|byte| !byte.is_ascii_whitespace()) + .count() + % 4 + == 0 + } +} + +/// Decodes `%XX` escapes. A `%` without two hex digits stays as written, as +/// browsers read a `data:` URI. +fn percent_decode(input: &[u8]) -> Vec { + let mut out = Vec::with_capacity(input.len()); + let mut index = 0; + while let Some(&byte) = input.get(index) { + if byte == b'%' + && let (Some(high), Some(low)) = ( + input.get(index + 1).and_then(|digit| hex_value(*digit)), + input.get(index + 2).and_then(|digit| hex_value(*digit)), + ) + { + out.push((high << 4) | low); + index += 3; + continue; + } + out.push(byte); + index += 1; + } + out +} + +fn hex_value(digit: u8) -> Option { + match digit { + b'0'..=b'9' => Some(digit - b'0'), + b'a'..=b'f' => Some(digit - b'a' + 10), + b'A'..=b'F' => Some(digit - b'A' + 10), + _ => None, + } +} + +/// Whether `byte` is written as itself in a percent-encoded payload. Quotes, +/// `&`, `<`, `>`, `=`, whitespace, parentheses, `\`, `,`, `#` and `%` are +/// always escaped, so the payload is safe in every context the module +/// rewrites and a `srcset` candidate never ends in a comma. +fn is_literal(byte: u8) -> bool { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'-' | b'.' + | b'_' + | b'~' + | b'!' + | b'$' + | b'*' + | b'+' + | b';' + | b':' + | b'@' + | b'/' + | b'?' + ) +} + +fn percent_encode_into(out: &mut String, bytes: &[u8]) { + const DIGITS: &[u8; 16] = b"0123456789ABCDEF"; + for &byte in bytes { + if is_literal(byte) { + out.push(char::from(byte)); + } else { + out.push('%'); + out.push(char::from(DIGITS[usize::from(byte >> 4)])); + out.push(char::from(DIGITS[usize::from(byte & 0x0f)])); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_media_data_uris_are_parsed() { + assert!(DataUri::parse("data:image/png;base64,iVBORw0KGgo=").is_some()); + assert!(DataUri::parse("DATA:Image/SVG+xml;charset=utf-8,%3Csvg%3E").is_some()); + assert!(DataUri::parse("data:font/woff2;base64,d09GMg==").is_some()); + for value in [ + "data:text/html,

x

", + "data:,plain", + "data:application/json;base64,e30=", + "https://cdn.example.com/a.png", + "data:image/png;base64", + ] { + assert_eq!(DataUri::parse(value), None, "should not sign {value}"); + } + } + + #[test] + fn base64_round_trips_exact_bytes_and_keeps_header_and_padding_style() { + let uri = DataUri::parse("Data:image/png;name=a.png;BASE64,AAEC/w==") + .expect("should parse a media data URI"); + assert_eq!(uri.media_type(), "image/png"); + assert_eq!(uri.declared_type(), "image/png;name=a.png"); + assert_eq!( + uri.decode().expect("should decode"), + vec![0, 1, 2, 255], + "should decode the exact bytes" + ); + let signed: Vec = (0..=255).collect(); + let encoded = uri.encode(&signed); + assert!( + encoded.starts_with("Data:image/png;name=a.png;BASE64,"), + "should keep the scheme and header as written: {encoded}" + ); + let reparsed = DataUri::parse(&encoded).expect("should reparse"); + assert_eq!( + reparsed.decode().expect("should decode"), + signed, + "should round-trip" + ); + + let unpadded = DataUri::parse("data:image/gif;base64,AAEC/w").expect("should parse"); + let encoded = unpadded.encode(&[1, 2]); + assert!(!encoded.ends_with('='), "should keep an unpadded form"); + assert_eq!( + DataUri::parse(&encoded) + .expect("should reparse") + .decode() + .expect("should decode"), + vec![1, 2] + ); + } + + #[test] + fn wrapped_base64_is_decoded_as_browsers_do() { + let uri = DataUri::parse("data:image/png;base64,AAEC\n /w==").expect("should parse"); + assert_eq!(uri.decode().expect("should decode"), vec![0, 1, 2, 255]); + let bad = DataUri::parse("data:image/png;base64,A*==").expect("should parse"); + assert_eq!(bad.decode(), Err(Malformed), "should refuse invalid base64"); + } + + #[test] + fn percent_encoded_svg_round_trips_and_is_context_safe() { + let original = "data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg'%3E%3C/svg%3E"; + let uri = DataUri::parse(original).expect("should parse"); + let decoded = uri.decode().expect("should decode"); + assert_eq!( + decoded, + b"".to_vec(), + "should decode escapes and keep literal bytes" + ); + let signed = b"(x), #&\\ \n100%\xff"; + let encoded = uri.encode(signed); + assert!( + !encoded[original.find(',').expect("comma") + 1..] + .bytes() + .any(|byte| matches!( + byte, + b'"' | b'\'' + | b'<' + | b'>' + | b'&' + | b'=' + | b'(' + | b')' + | b'\\' + | b',' + | b'#' + | b' ' + | b'\n' + )), + "should escape every context-sensitive byte: {encoded}" + ); + assert_eq!( + DataUri::parse(&encoded) + .expect("should reparse") + .decode() + .expect("should decode"), + signed.to_vec(), + "should round-trip exactly" + ); + } +} diff --git a/crates/provenance/encypher/src/embed.rs b/crates/provenance/encypher/src/embed.rs new file mode 100644 index 0000000000..58b811d8ad --- /dev/null +++ b/crates/provenance/encypher/src/embed.rs @@ -0,0 +1,523 @@ +use super::api::EmbeddingPlan; +use super::boundary::Boundary; +use super::text::visit_canonical_text; + +const MAGIC_PREFIX: &str = "\u{e01df}\u{e01e0}\u{e01e1}\u{e01e2}"; +const MANIFEST_META_PREFIX: &str = r#""#; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ApplyError { + Alignment, + TooLarge, +} + +/// Reports whether a string contains the four-character Encypher magic prefix. +#[must_use] +pub fn has_existing_markers(s: &str) -> bool { + s.contains(MAGIC_PREFIX) +} + +/// Applies a codepoint embedding plan and optional manifest insertion in one +/// exact-capacity output allocation. +/// +/// The plan must already have passed [`super::api::validate_plan`]. At an +/// equal source offset, plan markers precede the manifest in plan order. +pub(crate) fn apply_plan( + html: &str, + boundary: &Boundary, + plan: &EmbeddingPlan, + manifest: Option<(&str, usize)>, + max_output_bytes: usize, +) -> Result { + let marker_bytes = plan + .operations + .iter() + .try_fold(0usize, |total, operation| { + total.checked_add(operation.marker.len()) + }) + .ok_or(ApplyError::TooLarge)?; + let manifest_bytes = manifest + .map_or(Some(0), |(url, _)| manifest_meta_len(url)) + .ok_or(ApplyError::TooLarge)?; + let output_len = html + .len() + .checked_add(marker_bytes) + .and_then(|with_markers| with_markers.checked_add(manifest_bytes)) + .ok_or(ApplyError::TooLarge)?; + if output_len > max_output_bytes { + return Err(ApplyError::TooLarge); + } + + let boundary_html = html + .get(boundary.start..boundary.end) + .ok_or(ApplyError::Alignment)?; + let offsets_needed = match plan + .operations + .iter() + .filter_map(|operation| usize::try_from(operation.insert_after_index).ok()) + .max() + { + Some(index) => index.checked_add(1).ok_or(ApplyError::Alignment)?, + None => 0, + }; + let mut insertion_offsets = Vec::with_capacity(offsets_needed.min(boundary_html.len())); + let mut first_visible_start = None; + visit_canonical_text(boundary_html, |item| { + if first_visible_start.is_none() { + first_visible_start = Some(item.start); + } + if insertion_offsets.len() < offsets_needed { + insertion_offsets.push(item.end); + } + insertion_offsets.len() < offsets_needed + }); + let first_visible_start = first_visible_start.ok_or(ApplyError::Alignment)?; + + let mut insertions = Vec::with_capacity(plan.operations.len()); + for operation in &plan.operations { + let relative = if operation.insert_after_index == -1 { + first_visible_start + } else { + let index = + usize::try_from(operation.insert_after_index).map_err(|_| ApplyError::Alignment)?; + insertion_offsets + .get(index) + .copied() + .ok_or(ApplyError::Alignment)? + }; + let absolute = boundary + .start + .checked_add(relative) + .ok_or(ApplyError::Alignment)?; + insertions.push((absolute, operation.marker.as_str())); + } + // Stable sorting preserves API order for multiple markers at the same offset. + insertions.sort_by_key(|(offset, _)| *offset); + + let mut result = String::with_capacity(output_len); + let mut cursor = 0; + let mut manifest = manifest; + for (offset, marker) in insertions { + if manifest.is_some_and(|(_, manifest_offset)| manifest_offset < offset) { + let (url, manifest_offset) = manifest.take().ok_or(ApplyError::Alignment)?; + result.push_str( + html.get(cursor..manifest_offset) + .ok_or(ApplyError::Alignment)?, + ); + push_manifest_meta(&mut result, url); + cursor = manifest_offset; + } + result.push_str(html.get(cursor..offset).ok_or(ApplyError::Alignment)?); + result.push_str(marker); + cursor = offset; + } + if let Some((url, manifest_offset)) = manifest { + result.push_str( + html.get(cursor..manifest_offset) + .ok_or(ApplyError::Alignment)?, + ); + push_manifest_meta(&mut result, url); + cursor = manifest_offset; + } + result.push_str(html.get(cursor..).ok_or(ApplyError::Alignment)?); + Ok(result) +} + +/// Exact byte length of a manifest meta element after attribute escaping. +fn manifest_meta_len(manifest_url: &str) -> Option { + manifest_url.chars().try_fold( + MANIFEST_META_PREFIX.len() + MANIFEST_META_SUFFIX.len(), + |total, character| { + let bytes = match character { + '&' => 5, + '"' => 6, + '<' | '>' => 4, + _ => character.len_utf8(), + }; + total.checked_add(bytes) + }, + ) +} + +fn push_manifest_meta(output: &mut String, manifest_url: &str) { + output.push_str(MANIFEST_META_PREFIX); + for character in manifest_url.chars() { + match character { + '&' => output.push_str("&"), + '"' => output.push_str("""), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + _ => output.push(character), + } + } + output.push_str(MANIFEST_META_SUFFIX); +} + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + use super::*; + use crate::api::{EmbeddingPlan, PlanOperation}; + use crate::boundary::{Boundary, find_article_boundary}; + use crate::text::{document_metadata, visible_text}; + + fn apply(html: &str, boundary: &Boundary, plan: &EmbeddingPlan) -> Option { + apply_plan(html, boundary, plan, None, usize::MAX).ok() + } + + fn manifest_output(html: &str, manifest_url: &str) -> String { + let metadata = document_metadata(html); + if manifest_url.is_empty() || metadata.manifest_present { + return html.to_owned(); + } + let Some(offset) = metadata.head_close else { + return html.to_owned(); + }; + let mut output = + String::with_capacity(html.len() + manifest_meta_len(manifest_url).unwrap_or(0)); + output.push_str(html.get(..offset).expect("metadata offset should be valid")); + push_manifest_meta(&mut output, manifest_url); + output.push_str(html.get(offset..).expect("metadata offset should be valid")); + output + } + + #[test] + fn adversarial_ampersands_have_bounded_embedding_work() { + let html = format!("
{}
", "&".repeat(100_000)); + let boundary = find_article_boundary(&html, None).expect("article"); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_owned(), + }], + }; + let output = apply(&html, &boundary, &plan).expect("valid plan"); + assert_eq!( + output, + format!("
&\u{fe00}{}
", "&".repeat(99_999)) + ); + } + + #[test] + fn ordered_entity_replacements_preserve_original_insertion_spans() { + let html = "
&lt;|&#65;|&lt;
"; + let boundary = find_article_boundary(html, None).expect("article"); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: vec![ + PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_owned(), + }, + PlanOperation { + insert_after_index: 2, + marker: "\u{fe01}".to_owned(), + }, + PlanOperation { + insert_after_index: 4, + marker: "\u{fe02}".to_owned(), + }, + ], + }; + + assert_eq!( + visible_text( + html.get(boundary.start..boundary.end) + .expect("boundary should be valid"), + usize::MAX, + ) + .0, + "<|A|<" + ); + assert_eq!( + apply(html, &boundary, &plan), + Some( + "
&lt;\u{fe00}|&#65;\u{fe01}|&\u{fe02}lt;
".to_owned() + ) + ); + } + + #[test] + fn dense_plan_on_large_document_preserves_operation_order() { + let html = format!( + "
{}
", + "a".repeat(50_000), + "x".repeat(4_900_000) + ); + let boundary = find_article_boundary(&html, None).expect("article"); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: (0..20_000) + .rev() + .map(|index| PlanOperation { + insert_after_index: index, + marker: "\u{fe00}".to_owned(), + }) + .collect(), + }; + let output = apply(&html, &boundary, &plan).expect("valid dense plan"); + assert_eq!( + output, + format!( + "
{}{}
", + "a\u{fe00}".repeat(20_000), + "a".repeat(30_000), + "x".repeat(4_900_000) + ) + ); + } + + #[test] + fn sparse_plan_on_markup_dense_article_bounds_fragment_work() { + let html = format!("
{}
", "a".repeat(612_500)); + let boundary = find_article_boundary(&html, None).expect("article"); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: vec![ + PlanOperation { + insert_after_index: -1, + marker: "\u{fe00}".to_owned(), + }, + PlanOperation { + insert_after_index: 0, + marker: "\u{fe01}".to_owned(), + }, + PlanOperation { + insert_after_index: 0, + marker: "\u{fe02}".to_owned(), + }, + ], + }; + + let output = apply(&html, &boundary, &plan).expect("valid sparse plan"); + let marker_bytes = plan + .operations + .iter() + .map(|operation| operation.marker.len()) + .sum::(); + assert_eq!(output.len(), html.len() + marker_bytes); + assert!(output.starts_with("
\u{fe00}a\u{fe01}\u{fe02}")); + assert!(output.ends_with("
")); + } + + #[derive(Deserialize)] + struct FixtureFile { + cases: Vec, + } + + #[derive(Deserialize)] + struct FixtureCase { + id: String, + html: String, + override_selector: Option, + boundary: FixtureBoundary, + text: String, + text_rust: String, + text_divergence: Option, + plan: EmbeddingPlan, + expected_output: String, + invalid_plan: Option, + expected_invalid_output: Option, + manifest_url: String, + expected_meta_output_js: String, + expected_meta_output_rust: String, + meta_divergence: Option, + } + + #[derive(Deserialize)] + struct FixtureBoundary { + selector: String, + start: usize, + end: usize, + } + + fn fixtures() -> FixtureFile { + serde_json::from_str(include_str!("fixtures/parity.json")) + .expect("parity fixtures should deserialize") + } + + #[test] + fn groups_same_offset_in_plan_order() { + let html = "🐟
A B
"; + let start = html.find("
").expect("should find boundary"); + let boundary = Boundary { + start, + end: html.len(), + selector: "article".to_string(), + }; + let plan = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![ + PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_string(), + }, + PlanOperation { + insert_after_index: 0, + marker: "\u{fe01}".to_string(), + }, + ], + }; + assert_eq!( + apply(html, &boundary, &plan), + Some("🐟
A\u{fe00}\u{fe01} B
".to_string()) + ); + } + + #[test] + fn manifest_meta_uses_structural_head_close_and_escapes() { + let html = ""; + let output = manifest_output(html, "https://example.com/?a=1&b=\"x\""); + assert!(output.contains(" -->"#, + r#""#, + r#"
"#, + ] { + assert!( + !document_metadata(html).manifest_present, + "inert markup must not declare a manifest: {html}" + ); + } + assert!( + document_metadata( + r#""# + ) + .manifest_present + ); + } + + #[test] + fn invalid_less_than_and_declarations_do_not_hide_manifest_meta() { + let html = concat!( + "text < y ", + " remainder\">", + "", + r#""# + ); + assert!(document_metadata(html).manifest_present); + assert!( + !document_metadata( + r#"< meta name="c2pa-manifest-url">"# + ) + .manifest_present + ); + } + + #[test] + fn manifest_meta_does_not_activate_quoted_markup() { + let prefix = r#""#; + let html = format!("{prefix}
Visible
"); + assert_eq!( + manifest_output(&html, "javascript:alert(1)"), + format!( + "{prefix}
Visible
" + ) + ); + // The URL is inert metadata, not an href, script source or refresh target. + let script = + r#""#; + assert_eq!( + manifest_output( + &format!("{script}"), + "data:text/html," + ), + format!( + "{script}" + ) + ); + } + + #[test] + fn manifest_meta_ignores_quotes_inside_unquoted_values() { + // The browser reads `x"y` as an unquoted value, so the second meta's quoted + // content, including its ``, is attribute text. + let prefix = + r#""#; + let html = format!("{prefix}Visible"); + assert_eq!( + manifest_output(&html, "https://manifest.example.com/m"), + format!( + "{prefix}Visible" + ) + ); + } + + #[test] + fn edge_core_parity_fixtures_match() { + for fixture in fixtures().cases { + let boundary = + find_article_boundary(&fixture.html, fixture.override_selector.as_deref()) + .unwrap_or_else(|| panic!("{} should have a boundary", fixture.id)); + assert_eq!( + boundary.selector, fixture.boundary.selector, + "{}", + fixture.id + ); + assert_eq!(boundary.start, fixture.boundary.start, "{}", fixture.id); + assert_eq!(boundary.end, fixture.boundary.end, "{}", fixture.id); + let boundary_html = fixture + .html + .get(boundary.start..boundary.end) + .expect("fixture boundary should be valid"); + let text = visible_text(boundary_html, usize::MAX).0; + assert_eq!(text, fixture.text_rust, "{}", fixture.id); + if fixture.text_divergence.is_some() { + assert_ne!(text, fixture.text, "{}", fixture.id); + } else { + assert_eq!(text, fixture.text, "{}", fixture.id); + } + assert_eq!( + apply(&fixture.html, &boundary, &fixture.plan), + Some(fixture.expected_output), + "{}", + fixture.id + ); + if let Some(invalid_plan) = fixture.invalid_plan { + assert_eq!( + apply(&fixture.html, &boundary, &invalid_plan), + fixture.expected_invalid_output, + "{}", + fixture.id + ); + } + let meta = manifest_output(&fixture.html, &fixture.manifest_url); + assert_eq!(meta, fixture.expected_meta_output_rust, "{}", fixture.id); + if fixture.meta_divergence.is_some() { + assert_ne!(meta, fixture.expected_meta_output_js, "{}", fixture.id); + } else { + assert_eq!(meta, fixture.expected_meta_output_js, "{}", fixture.id); + } + } + } + + #[test] + fn every_fixture_byte_prefix_is_panic_free() { + for fixture in fixtures().cases { + for end in 0..=fixture.html.len() { + let prefix = String::from_utf8_lossy( + fixture + .html + .as_bytes() + .get(..end) + .expect("prefix range should be valid"), + ); + let _ = visible_text(&prefix, usize::MAX); + if let Some(boundary) = + find_article_boundary(&prefix, fixture.override_selector.as_deref()) + { + let _ = apply(&prefix, &boundary, &fixture.plan); + } + let _ = manifest_output(&prefix, &fixture.manifest_url); + } + } + } +} diff --git a/crates/provenance/encypher/src/embedded.rs b/crates/provenance/encypher/src/embedded.rs new file mode 100644 index 0000000000..b9f5bd4a89 --- /dev/null +++ b/crates/provenance/encypher/src/embedded.rs @@ -0,0 +1,1248 @@ +//! Media embedded in a document: `data:` URIs in HTML attributes, `srcset` +//! candidates, inline styles and stylesheets, and inline `` elements. +//! +//! Scanning is pure. It finds the exact byte range of each media `data:` +//! URI the module can rewrite without changing anything else, and the reason +//! for each one it cannot: an inline style a page security policy admits by +//! hash, or whose policies the module cannot see, a subresource integrity +//! value on the element, or an escape the module would have to decode. A +//! refused URI stays as written and is reported as unsigned. Inline SVG is +//! signed as the standalone SVG document +//! [`signable_svg`](crate::svg_equivalence::signable_svg) proves equivalent +//! to what the browser parsed, and only kept when the signer's writer added +//! its manifest carrier and changed nothing else. + +use crate::data_uri::DataUri; +use crate::processor::Reason; +use crate::text::{ + comment_end, find_byte, find_bytes, is_structural_element_start, raw_close_name_end, + starts_with_at, tag_end, take_tag_name, +}; + +/// Elements whose content the HTML tokenizer never reads as markup. Unlike +/// the text path, `noscript` content is scanned as markup, because readers +/// without scripts see the images inside it. +const RAW_TEXT: [&str; 9] = [ + "script", + "style", + "title", + "textarea", + "xmp", + "iframe", + "noembed", + "noframes", + "plaintext", +]; + +/// Where a found `data:` URI sits, which decides what may refuse its edit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Site { + /// A URL attribute or a `srcset` candidate. + Attribute, + /// A `style` attribute, which a page policy may admit by hash. + StyleAttribute, + /// A `` elements. + meta_policies: Vec<&'a str>, + /// A `` policy written in a way the module does not read. + unreadable_meta_policy: bool, +} + +impl<'a> Collector<'a> { + fn new(document: &'a str) -> Self { + Self { + document, + scan: Scan::default(), + inline: Vec::new(), + meta_policies: Vec::new(), + unreadable_meta_policy: false, + } + } + + /// Keep the URI at `range` when it is media the module can rewrite in + /// place, or report why it is left as written. + fn consider(&mut self, (start, end): (usize, usize), site: Site, integrity: bool) { + let Some(text) = self.document.get(start..end) else { + return; + }; + if DataUri::parse(text).is_none() { + return; + } + let refusal = if integrity { + Some(Reason::IntegrityAttribute) + } else if text.contains('\\') && site != Site::Attribute { + Some(Reason::CssEscape) + } else if text.contains('&') + && matches!( + site, + Site::Attribute | Site::StyleAttribute | Site::StyleElement { foreign: true, .. } + ) + { + // A character reference the module would have to decode, and + // could not write back as it was. + Some(Reason::CharacterReference) + } else { + None + }; + let found = Found { start, end }; + match (refusal, site) { + (Some(reason), _) => self.scan.refused.push(reason), + (None, Site::StyleAttribute | Site::StyleElement { .. }) => { + self.inline.push((found, site)); + } + (None, _) => self.scan.found.push(found), + } + } + + /// Each `srcset` candidate URL in the value at `range`. + fn srcset(&mut self, (start, end): (usize, usize), integrity: bool) { + let bytes = self.document.as_bytes(); + let mut cursor = start; + while cursor < end { + while cursor < end + && bytes + .get(cursor) + .is_some_and(|byte| byte.is_ascii_whitespace() || *byte == b',') + { + cursor += 1; + } + let url_start = cursor; + while cursor < end + && bytes + .get(cursor) + .is_some_and(|byte| !byte.is_ascii_whitespace()) + { + cursor += 1; + } + let mut url_end = cursor; + while url_end > url_start && bytes.get(url_end - 1) == Some(&b',') { + url_end -= 1; + } + if url_end > url_start { + self.consider((url_start, url_end), Site::Attribute, integrity); + } + if url_end < cursor { + continue; + } + // The descriptors run to the next comma. + while cursor < end && bytes.get(cursor).is_some_and(|byte| *byte != b',') { + cursor += 1; + } + } + } + + /// One start tag's media URIs. + fn tag(&mut self, name: &str, token: &'a str, offset: usize) { + let attributes = attributes(token, offset); + let integrity = attributes + .iter() + .any(|attribute| attribute.name.eq_ignore_ascii_case("integrity")); + if name.eq_ignore_ascii_case("meta") { + self.meta_policy(&attributes); + } + for attribute in &attributes { + let Some(range) = attribute.value else { + continue; + }; + if attribute.name.eq_ignore_ascii_case("style") { + self.css(range, Site::StyleAttribute); + } else if attribute + .name + .get(attribute.name.len().saturating_sub(6)..) + .is_some_and(|suffix| suffix.eq_ignore_ascii_case("srcset")) + { + self.srcset(range, integrity); + } else if is_url_attribute(attribute.name) { + let range = trimmed(self.document, range); + let starts_data = self + .document + .get(range.0..range.0.saturating_add(5).min(range.1)) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")); + if starts_data { + self.consider(range, Site::Attribute, integrity); + } + } + } + } + + /// Keeps the policy of a ``. + fn meta_policy(&mut self, attributes: &[Attribute<'_>]) { + let document = self.document; + let value = |expected: &str| { + attributes + .iter() + .find(|attribute| attribute.name.eq_ignore_ascii_case(expected)) + .and_then(|attribute| attribute.value) + .and_then(|(start, end)| document.get(start..end)) + }; + let is_policy = value("http-equiv") + .is_some_and(|name| name.trim().eq_ignore_ascii_case("content-security-policy")); + if !is_policy { + return; + } + match value("content") { + Some(policy) if !policy.contains('&') => self.meta_policies.push(policy), + _ => self.unreadable_meta_policy = true, + } + } + + /// The media URLs of the CSS at `range`. + fn css(&mut self, range: (usize, usize), site: Site) { + for uri in css_urls(self.document, range) { + self.consider(uri, site, false); + } + } + + /// Classify the original outer SVG before returning any descendant edit. + /// All media collected since its start belongs to this range. A protected + /// or incomplete SVG is kept byte-for-byte, including root attributes, + /// CSS and nested SVGs; it is not submitted for a new whole-asset signature. + /// Returns whether the range is editable; only an explicit outer closing + /// tag makes it eligible for whole-SVG signing. + fn finish_svg( + &mut self, + start: usize, + end: usize, + complete: bool, + (found, inline, refused): (usize, usize, usize), + ) -> bool { + let refusal = self + .document + .get(start..end) + .map_or( + Some(Reason::InlineSvgNotEquivalent), + crate::svg_equivalence::descendant_edit_refusal, + ) + .or_else(|| (!complete).then_some(Reason::InlineSvgNotEquivalent)); + if let Some(reason) = refusal { + self.scan.found.truncate(found); + self.inline.truncate(inline); + self.scan.refused.truncate(refused); + self.scan.refused.push(reason); + self.scan.protected.push((start, end)); + false + } else { + true + } + } + + /// The scan, with each inline style URI kept only where `policies` (the + /// response's policy header values, `None` when unknown) and the page's + /// `` policies leave its style editable. + fn finish(self, policies: Option<&[String]>) -> Scan { + let Self { + document, + mut scan, + inline, + meta_policies, + unreadable_meta_policy, + } = self; + let all: Vec<&str> = policies + .unwrap_or_default() + .iter() + .map(String::as_str) + .chain(meta_policies) + .flat_map(|value| value.split(',')) + .collect(); + for (found, site) in inline { + let refusal = if policies.is_none() || unreadable_meta_policy { + Some(Reason::CspUnknown) + } else { + let locked = match site { + Site::StyleElement { nonce, .. } => { + let nonce = nonce.and_then(|(start, end)| document.get(start..end)); + locks(&all, &["style-src-elem", "style-src", "default-src"], nonce) + } + _ => locks(&all, &["style-src-attr", "style-src", "default-src"], None), + }; + locked.then_some(Reason::CspStyleHash) + }; + match refusal { + Some(reason) => scan.refused.push(reason), + None => scan.found.push(found), + } + } + scan.found.sort_by_key(|found| found.start); + scan + } +} + +/// Whether any of `policies` admits inline styles, under the first directive +/// of `order` it names, by hash, so an edit would block the style. The +/// element's `nonce` value admits it whatever its text where the directive +/// lists that exact nonce; another nonce does not. +fn locks(policies: &[&str], order: &[&str], nonce: Option<&str>) -> bool { + policies.iter().any(|policy| { + let directives: Vec<(&str, &str)> = policy + .split(';') + .map(|directive| { + let directive = directive.trim(); + directive + .split_once(|ch: char| ch.is_ascii_whitespace()) + .unwrap_or((directive, "")) + }) + .collect(); + let Some(sources) = order.iter().find_map(|wanted| { + directives + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(wanted)) + .map(|(_, sources)| *sources) + }) else { + return false; + }; + let hashed = sources.split_ascii_whitespace().any(|source| { + ["'sha256-", "'sha384-", "'sha512-"].iter().any(|prefix| { + source + .get(..prefix.len()) + .is_some_and(|start| start.eq_ignore_ascii_case(prefix)) + }) + }); + // The `'nonce-` keyword is case-insensitive, its value is not. + let nonced = nonce.is_some_and(|value| { + !value.is_empty() + && sources.split_ascii_whitespace().any(|source| { + source + .get(..7) + .is_some_and(|start| start.eq_ignore_ascii_case("'nonce-")) + && source.get(7..).and_then(|rest| rest.strip_suffix('\'')) == Some(value) + }) + }); + hashed && !nonced + }) +} + +/// Whether the attribute `name` holds one URL a browser loads or links: +/// the standard URL attributes and the lazy-loading `data-*` ones that +/// scripts copy into them. A `data:` value in any other attribute, such as +/// `title`, `value` or `content`, is text and is left alone. +fn is_url_attribute(name: &str) -> bool { + [ + "src", + "href", + "xlink:href", + "poster", + "data", + "background", + "data-src", + "data-lazy-src", + "data-original", + "data-bg", + "data-background", + "data-poster", + "data-href", + ] + .iter() + .any(|candidate| name.eq_ignore_ascii_case(candidate)) +} + +/// The `Content-Security-Policy` and `-Report-Only` values of `headers`, as +/// [`scan_html`] reads them, or `None` when a value is not text and so +/// cannot be read. +pub(crate) fn content_security_policies(headers: &http::HeaderMap) -> Option> { + let mut policies = Vec::new(); + for name in [ + "content-security-policy", + "content-security-policy-report-only", + ] { + for value in headers.get_all(name) { + policies.push(value.to_str().ok()?.to_owned()); + } + } + Some(policies) +} + +/// Finds media in an HTML document. Inline style media is kept only when +/// `policies` (the response's policy header values, `None` when the module +/// cannot see them) and the page's `` policies admit no inline style +/// by hash, as an edit would block it. Original provenance-bearing or +/// structurally uncertain SVG ranges admit no edits, including descendants. +pub(crate) fn scan_html(document: &str, policies: Option<&[String]>) -> Scan { + let bytes = document.as_bytes(); + let mut collector = Collector::new(document); + let mut svg_depth = 0_usize; + let mut svg_start = 0_usize; + let mut svg_checkpoint = (0, 0, 0); + let mut cursor = 0_usize; + while cursor < bytes.len() { + if starts_with_at(bytes, cursor, b"\ + " + ); + let scan = scan_html(&document, None); + assert!(scan.is_empty(), "should find no media: {:?}", scan.found); + } + + #[test] + fn integrity_and_character_references_are_refused_not_dropped() { + let document = format!( + "\ + " + ); + let scan = scan_html(&document, None); + assert!(scan.found.is_empty(), "should rewrite neither"); + assert_eq!( + scan.refused, + vec![Reason::IntegrityAttribute, Reason::CharacterReference], + "should report each as unsigned" + ); + } + + #[test] + fn inline_style_edits_wait_for_known_policies_without_style_hashes() { + let document = format!( + "
\ + \ + " + ); + let unknown = scan_html(&document, None); + assert_eq!( + texts(&document, &unknown), + vec![PNG], + "should edit only the attribute" + ); + assert_eq!( + unknown.refused, + vec![Reason::CspUnknown, Reason::CspUnknown], + "should report both styles under an unseen policy" + ); + + let open = [String::from("img-src 'self' data:")]; + let scan = scan_html(&document, Some(&open)); + assert_eq!( + texts(&document, &scan), + vec![PNG; 3], + "should edit all, in order" + ); + assert!( + scan.found + .windows(2) + .all(|pair| pair[0].start < pair[1].start), + "should keep document order" + ); + + let hashed = [String::from("style-src 'sha256-abc' 'nonce-n'")]; + let scan = scan_html(&document, Some(&hashed)); + assert_eq!( + texts(&document, &scan).len(), + 2, + "should edit the image and the nonced element" + ); + assert_eq!(scan.refused, vec![Reason::CspStyleHash]); + + let meta = format!( + "{document}" + ); + let scan = scan_html(&meta, Some(&open)); + assert_eq!(texts(&meta, &scan), vec![PNG], "should honor a meta policy"); + } + + #[test] + fn only_the_style_s_own_nonce_admits_an_edit_under_a_hash_policy() { + let style = |nonce: &str| { + format!("") + }; + let policy = [String::from("style-src 'sha256-abc' 'NONCE-n0Nce'")]; + for (nonce, edited) in [ + ("n0Nce", true), + ("other", false), + ("n0nce", false), + ("", false), + ] { + let document = style(nonce); + let scan = scan_html(&document, Some(&policy)); + assert_eq!( + scan.found.len(), + usize::from(edited), + "nonce {nonce:?}: should edit only on the exact listed value" + ); + if !edited { + assert_eq!(scan.refused, vec![Reason::CspStyleHash], "nonce {nonce:?}"); + } + } + } + + #[test] + fn style_text_inside_inline_svg_is_markup_whose_references_are_decoded() { + let uri = "data:image/svg+xml,%3Csvg%3E&"; + let open = [String::from("img-src data:")]; + let in_svg = format!( + "" + ); + let scan = scan_html(&in_svg, Some(&open)); + assert!( + scan.found.is_empty(), + "should not rewrite a decoded reference" + ); + assert_eq!(scan.refused, vec![Reason::CharacterReference]); + + let in_html = format!(""); + let scan = scan_html(&in_html, Some(&open)); + assert_eq!( + texts(&in_html, &scan), + vec![uri], + "an HTML style's text is raw, so its `&` is literal" + ); + + let script = format!( + "" + ); + let scan = scan_html(&script, Some(&open)); + assert_eq!( + texts(&script, &scan), + vec![PNG], + "should keep scanning after an SVG script" + ); + } + + #[test] + fn only_url_attributes_are_rewritten() { + let document = format!( + "\ + \"{PNG}\"\ +
" + ); + let scan = scan_html(&document, None); + let found: Vec = scan.found.iter().map(|found| found.start).collect(); + let lazy = document.find("data-lazy-src=\"").expect("lazy") + "data-lazy-src=\"".len(); + let background = document.find("data-bg=\"").expect("bg") + "data-bg=\"".len(); + let object = document.find("data=\"data:").expect("object") + "data=\"".len(); + assert_eq!( + found, + vec![lazy, background, object], + "should rewrite lazy, background and object URLs, never text attributes" + ); + } + + #[test] + fn policies_are_read_from_both_policy_headers() { + let mut headers = http::HeaderMap::new(); + headers.append( + "content-security-policy", + http::HeaderValue::from_static("img-src data:"), + ); + headers.append( + "content-security-policy-report-only", + http::HeaderValue::from_static("style-src 'sha256-abc'"), + ); + assert_eq!( + content_security_policies(&headers), + Some(vec![ + "img-src data:".to_owned(), + "style-src 'sha256-abc'".to_owned() + ]) + ); + } + + #[test] + fn stylesheets_find_url_and_image_set_media_outside_comments() { + let document = format!( + ".a{{background:url({PNG})}}/* url({PNG}) */.b{{background:url( \"{PNG}\" )}}\ + .c{{background-image:image-set(\"{PNG}\" 1x, url({PNG}) 2x)}}\ + .d{{background:url(data:image/png;base64,iVBO\\52w0KGgo=)}}.e{{content:\"{PNG}\"}}" + ); + let scan = scan_stylesheet(&document); + assert_eq!( + texts(&document, &scan), + vec![PNG; 4], + "should find each media URL" + ); + assert_eq!( + scan.refused, + vec![Reason::CssEscape], + "should refuse the escaped URL" + ); + } + + #[test] + fn outermost_inline_svgs_are_found_with_exact_ranges() { + let svg = + ""; + let document = format!("

a

{svg}

b

"); + let scan = scan_html(&document, None); + assert_eq!(scan.svgs.len(), 1, "should find one outermost element"); + let (start, end) = scan.svgs[0]; + assert_eq!(&document[start..end], svg, "should cover the exact bytes"); + } + + #[test] + fn self_closing_svg_attributes_are_scanned_without_whole_asset_signing() { + let document = format!(""); + let scan = scan_html(&document, Some(&[])); + assert_eq!(texts(&document, &scan), [PNG]); + assert!(scan.svgs.is_empty()); + assert!(scan.refused.is_empty()); + } + + #[test] + fn original_svg_provenance_protects_attributes_css_and_nested_svg_ranges() { + for provenance in [ + CARRIER, + "", + "https://example.com/m", + "", + "", + "QUJD", + ] { + let svg = format!( + "{ROOT}{provenance}\ + \ + " + ); + let document = format!("{svg}"); + let scan = scan_html(&document, Some(&[])); + assert_eq!(texts(&document, &scan), [PNG, PNG], "{provenance}"); + assert!(scan.svgs.is_empty(), "must not resign {provenance}"); + assert_eq!( + scan.refused, + [Reason::SvgExistingProvenance], + "{provenance}" + ); + } + } + + #[test] + fn unsigned_svg_descendants_remain_editable_without_a_false_caption_match() { + let document = format!( + "{ROOT}Signed with C2PA\ + \ + " + ); + let scan = scan_html(&document, Some(&[])); + assert_eq!(texts(&document, &scan), [PNG; 3]); + assert_eq!(scan.svgs, [(0, document.len())]); + assert!(scan.refused.is_empty()); + } + + #[test] + fn incomplete_or_uninspectable_svg_ranges_fail_closed_before_child_edits() { + for svg in [ + format!("{ROOT}{CARRIER}"), + format!("{ROOT}"), + format!("{ROOT}

breakout

"), + format!( + "{ROOT}{}", + "".repeat(10_000) + ), + format!(""), + ] { + let document = format!("{svg}"); + let scan = scan_html(&document, Some(&[])); + assert_eq!(texts(&document, &scan), [PNG], "{svg}"); + assert!(scan.svgs.is_empty(), "must not resign {svg}"); + assert_eq!(scan.refused.len(), 1); + } + } + + const ROOT: &str = ""; + const BODY: &str = ""; + const CARRIER: &str = "QUJD+/=="; + + #[test] + fn the_writer_s_carrier_after_the_root_start_tag_keeps_the_svg() { + let original = format!("{ROOT}{BODY}"); + let signed = format!("{ROOT}{CARRIER}{BODY}"); + assert_eq!( + verify_svg_carrier(original.as_bytes(), signed.as_bytes()), + Ok(()), + "should accept the writer's carrier" + ); + let declared = format!("\n{ROOT}{BODY}"); + let signed = format!("\n{ROOT}{CARRIER}{BODY}"); + assert_eq!( + verify_svg_carrier(declared.as_bytes(), signed.as_bytes()), + Ok(()), + "should find the root after a declaration and a comment" + ); + } + + #[test] + fn a_carrier_the_source_held_is_replaced_and_nothing_else() { + let old_element = "T0xE"; + for old in [old_element, ""] { + let original = format!("{ROOT}{old}{BODY}"); + let signed = format!("{ROOT}{CARRIER}{BODY}"); + assert_eq!( + verify_svg_carrier(original.as_bytes(), signed.as_bytes()), + Ok(()), + "should accept replacing {old}" + ); + } + } + + #[test] + fn any_other_change_to_the_svg_is_refused() { + let original = format!("{ROOT}{BODY}"); + let altered = [ + format!("{ROOT}{CARRIER}{BODY}").replace("currentColor", "#000"), + format!("{ROOT}{}{CARRIER}", BODY.replace("", "")) + "", + format!("{ROOT}{BODY}"), + format!("{ROOT}{}{BODY}", CARRIER.replace("QUJD+/==", "QU JD")), + format!("{ROOT} {CARRIER}{BODY}"), + format!("{ROOT}{BODY}"), + original.clone(), + ]; + for signed in altered { + assert_eq!( + verify_svg_carrier(original.as_bytes(), signed.as_bytes()), + Err(Reason::InlineSvgAltered), + "should refuse {signed}" + ); + } + } + + #[test] + fn splice_replaces_exact_ranges_within_budget() { + let document = ""; + let edits = vec![(10, 11, "xx".to_owned()), (23, 24, "y".to_owned())]; + assert_eq!( + splice(document, &edits, 100).as_deref(), + Some("") + ); + assert_eq!( + splice(document, &edits, document.len()), + None, + "should honor the budget" + ); + } +} diff --git a/crates/provenance/encypher/src/fixtures/parity.json b/crates/provenance/encypher/src/fixtures/parity.json new file mode 100644 index 0000000000..8487676317 --- /dev/null +++ b/crates/provenance/encypher/src/fixtures/parity.json @@ -0,0 +1,848 @@ +{ + "generated_from": [ + "integrations/edge-core/boundary.js", + "integrations/edge-core/fragments.js", + "integrations/edge-core/embed.js" + ], + "fixed_plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "cases": [ + { + "id": "override-multibyte-prefix", + "html": "Café & News🙂前

Alpha & beta.

", + "override_selector": "[data=story]", + "boundary": { + "selector": "override:[data=story]", + "start": 78, + "end": 126 + }, + "text": "Alpha & beta.", + "text_rust": "Alpha & beta.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Café & News🙂前

A︀︁lpha & beta.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Café & News🙂前

Alpha & beta.

", + "expected_meta_output_rust": "Café & News🙂前

Alpha & beta.

", + "meta_divergence": null + }, + { + "id": "article-most-paragraphs", + "html": "Two articles

Short.

First paragraph.

Second paragraph.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 84, + "end": 159 + }, + "text": "First paragraph. Second paragraph .", + "text_rust": "First paragraph. Second paragraph .", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Two articles

Short.

F︀︁irst paragraph.

Second paragraph.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Two articles

Short.

First paragraph.

Second paragraph.

", + "expected_meta_output_rust": "Two articles

Short.

First paragraph.

Second paragraph.

", + "meta_divergence": null + }, + { + "id": "cms-class-entities", + "html": "

Fish & chips — © 🐟.

", + "override_selector": null, + "boundary": { + "selector": "class:entry-content", + "start": 25, + "end": 123 + }, + "text": "Fish & chips — © 🐟.", + "text_rust": "Fish & chips — © 🐟.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

F︀︁ish & chips — © 🐟.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Fish & chips — © 🐟.

", + "expected_meta_output_rust": "

Fish & chips — © 🐟.

", + "meta_divergence": null + }, + { + "id": "itemprop-nbsp", + "html": "

One two three.

", + "override_selector": null, + "boundary": { + "selector": "itemprop:articleBody", + "start": 25, + "end": 94 + }, + "text": "One two three.", + "text_rust": "One two three.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁ne two three.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

One two three.

", + "expected_meta_output_rust": "

One two three.

", + "meta_divergence": null + }, + { + "id": "main-inline-tags", + "html": "

Inline

Hello fictional example.com.

", + "override_selector": null, + "boundary": { + "selector": "main", + "start": 25, + "end": 105 + }, + "text": "Inline Hello fictional example.com.", + "text_rust": "Inline Hello fictional example.com.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

I︀︁nline

Hello fictional example.com.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Inline

Hello fictional example.com.

", + "expected_meta_output_rust": "

Inline

Hello fictional example.com.

", + "meta_divergence": null + }, + { + "id": "role-main", + "html": "

Role based article boundary.

", + "override_selector": null, + "boundary": { + "selector": "role:main", + "start": 25, + "end": 83 + }, + "text": "Role based article boundary.", + "text_rust": "Role based article boundary.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

R︀︁ole based article boundary.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Role based article boundary.

", + "expected_meta_output_rust": "

Role based article boundary.

", + "meta_divergence": null + }, + { + "id": "paragraph-cluster", + "html": "

One.

Two.

Three.

Four.

Five.

", + "override_selector": null, + "boundary": { + "selector": "p-cluster", + "start": 25, + "end": 109 + }, + "text": "One. Two. Three. Four. Five.", + "text_rust": "One. Two. Three. Four. Five.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁ne.

Two.

Three.

Four.

Five.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

One.

Two.

Three.

Four.

Five.

", + "expected_meta_output_rust": "

One.

Two.

Three.

Four.

Five.

", + "meta_divergence": null + }, + { + "id": "body-fallback", + "html": "

Fallback

Body article text.", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 19, + "end": 80 + }, + "text": "Fallback Body article text.", + "text_rust": "Fallback Body article text.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

F︀︁allback

Body article text.", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Fallback

Body article text.", + "expected_meta_output_rust": "

Fallback

Body article text.", + "meta_divergence": null + }, + { + "id": "nested-same-name-first-close", + "html": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 90 + }, + "text": "Outer start. Inner text.", + "text_rust": "Outer start. Inner text.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁uter start.

Inner text.

Outer tail omitted by parity.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "expected_meta_output_rust": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "meta_divergence": null + }, + { + "id": "head-close-inside-script", + "html": "Script case

Script boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 86, + "end": 131 + }, + "text": "Script boundary fallback.", + "text_rust": "Script boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Script case

S︀︁cript boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Script case

Script boundary fallback.

", + "expected_meta_output_rust": "Script case

Script boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside script text; Rust inserts before the structural closing head tag." + }, + { + "id": "head-close-inside-comment", + "html": "Comment case

Comment boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 80, + "end": 126 + }, + "text": "Comment boundary fallback.", + "text_rust": "Comment boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Comment case

C︀︁omment boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Comment case

Comment boundary fallback.

", + "expected_meta_output_rust": "Comment case

Comment boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside a comment; Rust inserts before the structural closing head tag." + }, + { + "id": "head-close-inside-title", + "html": "Headline </head> text

Title boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 55, + "end": 99 + }, + "text": "Title boundary fallback.", + "text_rust": "Title boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Headline </head> text

T︀︁itle boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Headline <meta name=\"c2pa-manifest-url\" content=\"https://manifest.example.com/c2pa?id=fictional&kind="article"\"></head> text

Title boundary fallback.

", + "expected_meta_output_rust": "Headline </head> text

Title boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside title text; Rust never searches raw-text or RCDATA content for the closing head tag." + }, + { + "id": "javascript-whitespace-set", + "html": "

Alpha 
beta\t gamma.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 77 + }, + "text": "Alpha beta gamma.", + "text_rust": "Alpha beta gamma.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

A︀︁lpha 
beta\t gamma.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Alpha 
beta\t gamma.

", + "expected_meta_output_rust": "

Alpha 
beta\t gamma.

", + "meta_divergence": null + }, + { + "id": "skip-raw-elements-and-numeric-entities", + "html": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 162 + }, + "text": "Visible A. Tail B.", + "text_rust": "Visible A. Tail B.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

V︀︁isible A.

Hidden code.
Hidden pre.

Tail B.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "expected_meta_output_rust": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "meta_divergence": null + }, + { + "id": "only-the-seven-skip-elements-are-hidden", + "html": "

Lead text.

Inline title.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 166 + }, + "text": "Lead text. Draft note. Frame fallback. Inline title. After.", + "text_rust": "Lead text. Draft note. Frame fallback. Inline title. After.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

Inline title.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

Inline title.

After.

", + "expected_meta_output_rust": "

Lead text.

Inline title.

After.

", + "meta_divergence": null + }, + { + "id": "self-closing-svg-hides-nothing", + "html": "

Lead text.

After icon.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 103 + }, + "text": "Lead text. After icon.", + "text_rust": "Lead text. After icon.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After icon.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After icon.

", + "expected_meta_output_rust": "

Lead text.

After icon.

", + "meta_divergence": null + }, + { + "id": "quoted-gt-in-attribute", + "html": "

Lead text.

\"a b\">Tail text.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 95 + }, + "text": "Lead text. b\">Tail text.", + "text_rust": "Lead text. Tail text.", + "text_divergence": "JS ends a tag at the first \">\", so the rest of the alt value becomes text; Rust ends it at the first \">\" outside a quoted attribute value.", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

\"a b\">Tail text.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

\"a b\">Tail text.

", + "expected_meta_output_rust": "

Lead text.

\"a b\">Tail text.

", + "meta_divergence": null + }, + { + "id": "nested-same-name-ignored-element", + "html": "

Lead text.

outer inner tail

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 116 + }, + "text": "Lead text. After.", + "text_rust": "Lead text. tail After.", + "text_divergence": "JS counts nested start and end tags and drops the outer tail; Rust closes the ignored region at the first matching end tag.", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

outer inner tail

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

outer inner tail

After.

", + "expected_meta_output_rust": "

Lead text.

outer inner tail

After.

", + "meta_divergence": null + }, + { + "id": "self-closing-script-stays-raw-text", + "html": "

Lead text.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 107 + }, + "text": "Lead text. var hidden = 1; After.", + "text_rust": "Lead text. After.", + "text_divergence": "JS treats \".", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After.

", + "expected_meta_output_rust": "

Lead text.

After.

", + "meta_divergence": null + }, + { + "id": "end-tag-with-attributes-closes-ignored-element", + "html": "

Lead text.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 132 + }, + "text": "Lead text. After.", + "text_rust": "Lead text. After.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After.

", + "expected_meta_output_rust": "

Lead text.

After.

", + "meta_divergence": null + }, + { + "id": "abrupt-and-bang-comment-ends", + "html": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 131 + }, + "text": "Lead text. Shown two.", + "text_rust": "Lead text. Shown one. Shown two. Shown three. After.", + "text_divergence": "JS ends every comment at the next \"-->\"; Rust, like browsers, treats \"\" and \"\" as whole comments and also ends a comment at \"--!>\".", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

Shown one.Shown two.Shown three.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "expected_meta_output_rust": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "meta_divergence": null + }, + { + "id": "out-of-range-index", + "html": "

Small boundary.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 66 + }, + "text": "Small boundary.", + "text_rust": "Small boundary.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

S︀︁mall boundary.

", + "invalid_plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": 15, + "marker": "︀" + } + ] + }, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Small boundary.

", + "expected_meta_output_rust": "

Small boundary.

", + "meta_divergence": null + } + ] +} diff --git a/crates/provenance/encypher/src/lib.rs b/crates/provenance/encypher/src/lib.rs new file mode 100644 index 0000000000..e5ec65909b --- /dev/null +++ b/crates/provenance/encypher/src/lib.rs @@ -0,0 +1,585 @@ +//! Encypher provenance for article pages and their media, `provenance.encypher`. +//! +//! A serve middleware finds the article in each HTML page a `[[serve]]` entry +//! names it for, sends its visible text to the Encypher signing API, and +//! embeds the returned invisible markers and a manifest meta tag. Plans are +//! cached in the runtime KV store, bound to a short-lived server policy. +//! Every failure passes the page through unchanged. +//! +//! Media is signed as its exact bytes through the Encypher media endpoint, +//! under the same policy and worker token: images, audio, video and fonts +//! core captures on the publisher and asset proxy paths, through the +//! module's media signer; media `data:` URIs in the same pages, and their +//! inline SVG as the standalone SVG document proven to parse to the tree the +//! browser built, which is the original bytes where they already are one; +//! and media `data:` URIs in the stylesheets under each `path` a +//! `[[serve]]` entry names `provenance.encypher.stylesheet` for, which the +//! publisher asserts no page loads with `integrity`. Stylesheets under no +//! such path are left as they are. A media item that cannot be signed is +//! delivered as it was and reported unsigned, never as signed. +//! +//! The middleware's preparation applies the request gates with no I/O: only +//! a `GET` of a `200` page on the configured domain, whose origin did not +//! mark it as one reader's, is held. The signing work runs in the stream +//! processor's asynchronous finish, which core awaits at the end of the +//! document. The module requires no permission and reads no gated value. It +//! sends the visible text of the page's article, the page address without +//! its query, and the bytes of the media it signs. A page whose origin set a +//! cookie or forbade shared caching is never sent, but the module cannot +//! tell a personalized article that carries neither, so a deployment places +//! it only on public article routes and has its origin mark every +//! personalized response private. +//! +//! Independently of signing admission, the applicable page middleware protects +//! original provenance-bearing or structurally uncertain inline SVG ranges. +//! Core holds the bounded original before its own rewrites, keeps those ranges +//! opaque through fetch/serve changes and restores their exact bytes before +//! publishing a template or reader response. The same classifier decides both +//! this protection and the module's descendant-edit refusals. + +use std::any::Any; +use std::sync::Arc; + +use error_stack::Report; +use serde::Deserialize; +use validator::{Validate, ValidationError}; + +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationRegistration, ModuleSecretSetting, SuppliedMiddleware, +}; +use trusted_server_core::middleware::{ + HTML_MEDIA_TYPE, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, Preparation, +}; +use trusted_server_core::module_context::{FromModuleContext as _, ModuleCall, ModuleRequest}; +use trusted_server_core::redacted::Redacted; +use trusted_server_core::settings::{IntegrationConfig, Settings}; + +mod api; +mod boundary; +mod data_uri; +mod embed; +mod embedded; +mod media_api; +mod prepared_probe; +mod processor; +mod svg_equivalence; +mod text; + +use processor::{ + EncypherMedia, PageRequest, Signer, document_processor, is_excluded_path, is_excluded_prefix, + stylesheet_processor, +}; + +/// The id the module's registration and its API backend carry. +const INTEGRATION_ID: &str = "provenance_encypher"; + +/// The name `[provenance] module` selects the module by and a `[[serve]]` +/// entry names its middleware by, from this crate's folder. +pub const MODULE: &str = "provenance.encypher"; + +/// The name a `[[serve]]` entry for stylesheets writes, which signs the media +/// `data:` URIs a stylesheet embeds. +pub const STYLESHEET_MIDDLEWARE: &str = "provenance.encypher.stylesheet"; + +/// The media type [`STYLESHEET_MIDDLEWARE`] works on. +const STYLESHEET_MEDIA_TYPE: &str = "text/css"; + +/// `[provenance.encypher]`, the module's settings. +#[derive(Debug, Clone, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +#[validate(schema(function = "validate_hold_exceeds_timeout"))] +pub struct EncypherConfig { + /// Encypher API origin, for example `https://api.example.com`. Must be https. + #[validate(custom(function = validate_api_base_url))] + pub api_base_url: String, + /// Claimed publisher domain: a bare lowercase hostname without `www.`. + #[validate(custom(function = validate_domain))] + pub domain: String, + /// The name of the secret-store entry holding the `cwk_` worker + /// credential, resolved to the credential itself when settings load. + pub worker_token: Redacted, + /// Optional article selector override: one unescaped `.class`, an exact + /// `[attr=value]` (also single- or double-quoted), or an ASCII tag name. + /// Empty means no override; a valid selector absent from a page keeps + /// the detection fallback. At most 91 characters, so + /// `override:{selector}` fits the API's 100-character limit. + #[serde(default)] + #[validate(length(max = 91), custom(function = validate_article_selector))] + pub article_selector: String, + /// Minimum visible article text, in Unicode scalars. + #[serde(default = "default_min_text_length")] + #[validate(range(min = 50, max = 50_000))] + pub min_text_length: usize, + /// Largest decoded document held for signing. Capped at + /// `publisher.max_buffered_body_bytes`. + #[serde(default = "default_max_document_bytes")] + #[validate(range(min = 1024))] + pub max_document_bytes: usize, + /// First-byte timeout for each policy or sign call, in milliseconds. The + /// only bound on a hung API on a runtime without a timer. + #[serde(default = "default_timeout_ms")] + #[validate(range(min = 250, max = 10_000))] + pub timeout_ms: u64, + /// Longest one document is held for signing, in milliseconds, on a + /// runtime with a timer. Must exceed `timeout_ms`. + #[serde(default = "default_max_hold_ms")] + #[validate(range(min = 500, max = 15_000))] + pub max_hold_ms: u64, + /// Lifetime of a cached signing plan, in seconds. + #[serde(default = "default_plan_ttl_seconds")] + #[validate(range(min = 60, max = 31_536_000))] + pub plan_ttl_seconds: u64, + /// Lifetime of the server-authoritative policy, in seconds. + #[serde(default = "default_policy_ttl_seconds")] + #[validate(range(min = 60, max = 300))] + pub policy_ttl_seconds: u64, + /// Lifetime of a cached signing failure, in seconds. + #[serde(default = "default_negative_ttl_seconds")] + #[validate(range(min = 60, max = 86_400))] + pub negative_ttl_seconds: u64, + /// Largest media item signed, in bytes: a captured image or a decoded + /// embedded item. Larger media is delivered unsigned. Bounded by what the + /// endpoint embeds as source data when the source has no safe address. + #[serde(default = "default_max_media_bytes")] + #[validate(range(min = 1024, max = 8_000_000))] + pub max_media_bytes: usize, + /// Maximum uncached media-signing calls in one HTML or stylesheet pass. + /// Cached assets do not consume calls. The document's existing time and + /// output bounds still apply independently. + #[serde(default = "default_max_media_sign_calls_per_document")] + #[validate(range(min = 1, max = 128))] + pub max_media_sign_calls_per_document: usize, + /// Largest media signed through the prepared lane, in bytes: hashed in + /// bounded chunks where it is delivered, never uploaded or held whole, + /// and given a remote manifest its bytes are left unchanged by. `0`, the + /// default, keeps the lane off, so no deployment starts hashing large + /// files on its own; turning it on also needs the signed probe release + /// and the prepared endpoints. Otherwise at least 16 MiB, and at most the + /// prepared service's largest source. Media between `max_media_bytes` + /// and this is the lane's. A bounded example is `1073741824` (1 GiB) + /// with `max_large_media_ms = 60000`. + #[serde(default = "default_max_large_media_bytes")] + #[validate(custom(function = validate_max_large_media_bytes))] + pub max_large_media_bytes: u64, + /// Longest one large source may take, in milliseconds: core's hash of + /// the whole source and the module's probe and API calls together. A + /// source not signed in time is delivered unsigned. + #[serde(default = "default_max_large_media_ms")] + #[validate(range(min = 5_000, max = 600_000))] + pub max_large_media_ms: u64, +} + +/// The prepared service's largest source, in bytes: the largest integer a +/// JSON number carries exactly, less the room it keeps for a manifest. +const MAX_PREPARED_SOURCE_BYTES: u64 = 9_007_199_246_352_383; +/// The smallest non-zero `max_large_media_bytes`, above every captured size. +const MIN_LARGE_MEDIA_BYTES: u64 = 16 * 1024 * 1024; + +impl IntegrationConfig for EncypherConfig {} + +fn default_min_text_length() -> usize { + 50 +} + +fn default_max_document_bytes() -> usize { + 5_000_000 +} + +fn default_timeout_ms() -> u64 { + 3_000 +} + +fn default_max_hold_ms() -> u64 { + 5_000 +} + +fn default_plan_ttl_seconds() -> u64 { + 604_800 +} + +fn default_policy_ttl_seconds() -> u64 { + 300 +} + +fn default_negative_ttl_seconds() -> u64 { + 300 +} + +fn default_max_media_bytes() -> usize { + 5_000_000 +} + +fn default_max_media_sign_calls_per_document() -> usize { + 8 +} + +fn default_max_large_media_bytes() -> u64 { + 0 +} + +fn default_max_large_media_ms() -> u64 { + 60_000 +} + +fn validate_max_large_media_bytes(value: u64) -> Result<(), ValidationError> { + if value == 0 || (MIN_LARGE_MEDIA_BYTES..=MAX_PREPARED_SOURCE_BYTES).contains(&value) { + Ok(()) + } else { + Err(ValidationError::new("encypher_max_large_media_bytes")) + } +} + +fn validate_article_selector(value: &str) -> Result<(), ValidationError> { + if value.is_empty() || boundary::is_valid_selector(value) { + Ok(()) + } else { + Err(ValidationError::new("encypher_article_selector")) + } +} + +fn validate_api_base_url(value: &str) -> Result<(), ValidationError> { + let parsed = + url::Url::parse(value).map_err(|_| ValidationError::new("encypher_api_base_url"))?; + if parsed.scheme() != "https" + || parsed.host_str().is_none_or(str::is_empty) + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(ValidationError::new("encypher_api_base_url")); + } + Ok(()) +} + +fn validate_domain(value: &str) -> Result<(), ValidationError> { + let valid_label = |label: &str| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + }; + let valid = value.len() <= 253 + && value.contains('.') + && !value.starts_with("www.") + && value.split('.').all(valid_label); + if valid { + Ok(()) + } else { + Err(ValidationError::new("encypher_domain")) + } +} + +fn validate_hold_exceeds_timeout(config: &EncypherConfig) -> Result<(), ValidationError> { + if config.max_hold_ms > config.timeout_ms { + Ok(()) + } else { + Err(ValidationError::new("encypher_max_hold_ms")) + } +} + +/// The worker token, named in `[provenance.encypher]` by its entry in the +/// secret store, which is in use whenever the module is selected. +const SECRET_SETTINGS: &[ModuleSecretSetting] = &[ModuleSecretSetting { + path: &["worker_token"], + in_use: always_in_use, +}]; + +fn always_in_use(_table: &serde_json::Map) -> bool { + true +} + +/// The changes the module supplies, as `[[serve]]` entries name them: the +/// page's article and embedded media, and a stylesheet's embedded media. +const SUPPLIED_MIDDLEWARE: &[SuppliedMiddleware] = &[ + SuppliedMiddleware::on_html(MODULE, MiddlewarePhase::Serve), + SuppliedMiddleware::on( + STYLESHEET_MIDDLEWARE, + MiddlewarePhase::Serve, + STYLESHEET_MEDIA_TYPE, + ), +]; + +/// The builder a deployment installs to offer this module. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(INTEGRATION_ID, env!("CARGO_PKG_NAME"), register, validate) + .with_module_name(MODULE) + .with_secret_settings(SECRET_SETTINGS) + .with_middleware(SUPPLIED_MIDDLEWARE) +} + +/// Refuses a document that selects the module and names its middleware in no +/// `[[serve]]` entry, where no page would ever be signed. +fn require_an_entry(settings: &Settings) -> Result<(), Report> { + let placed = settings + .serve + .entries() + .iter() + .any(|entry| entry.middleware.iter().any(|name| name == MODULE)); + if placed { + return Ok(()); + } + Err(Report::new(TrustedServerError::Settings { + message: format!( + "[provenance] selects `encypher`, but no [[serve]] entry names `{MODULE}`, so no \ + page would be signed. Add `{MODULE}` to the [[serve]] entry that covers the \ + article pages, with a `path` for each section to sign" + ), + })) +} + +/// Refuses a `[[serve]]` entry that names the stylesheet middleware without +/// a `path` narrower than the whole site. +/// +/// Signing a stylesheet's media changes its bytes, which breaks a page that +/// loads it with a subresource integrity value, and the middleware cannot +/// see the page that loads it. So it runs only on stylesheets the publisher +/// names by path, asserting none of them is loaded with `integrity`; a bare +/// `text/css` entry, or one for `/`, would rewrite every stylesheet. +fn require_scoped_stylesheet_entries( + settings: &Settings, +) -> Result<(), Report> { + let unscoped = settings.serve.entries().iter().any(|entry| { + entry + .middleware + .iter() + .any(|name| name == STYLESHEET_MIDDLEWARE) + && entry + .path + .as_deref() + .is_none_or(|path| path.is_empty() || path == "/") + }); + if !unscoped { + return Ok(()); + } + Err(Report::new(TrustedServerError::Settings { + message: format!( + "a [[serve]] entry names `{STYLESHEET_MIDDLEWARE}` for every path. Signing a \ + stylesheet's embedded media changes its bytes and breaks a page that loads it \ + with `integrity`, so give the entry a `path` for the stylesheets no page loads \ + with `integrity`" + ), + })) +} + +/// The module's settings when a section selects it. +fn config(settings: &Settings) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + require_an_entry(settings)?; + require_scoped_stylesheet_entries(settings)?; + Ok(Some(config)) +} + +/// Validates the module's table and reports whether a section selects the +/// module. +/// +/// # Errors +/// +/// Returns an error when the table cannot be read or fails its validation, or +/// when no `[[serve]]` entry names the middleware. +pub fn validate(settings: &Settings) -> Result> { + Ok(config(settings)?.is_some()) +} + +/// Registers the middleware and the media signer when a section selects the +/// module and its worker token is not blank. +/// +/// # Errors +/// +/// Returns the errors [`validate`] returns. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = config(settings)? else { + return Ok(None); + }; + if config.worker_token.expose().trim().is_empty() { + log::warn!("encypher: not registered, because the worker token is empty"); + return Ok(None); + } + log::info!("encypher: registered for domain={}", config.domain); + let max_document_bytes = config + .max_document_bytes + .min(settings.publisher.max_buffered_body_bytes); + let signer = Arc::new(Signer::new(config, max_document_bytes)); + Ok(Some( + IntegrationRegistration::builder(INTEGRATION_ID) + .without_js() + .with_middleware(Arc::new(Encypher { + signer: Arc::clone(&signer), + })) + .with_middleware(Arc::new(EncypherStylesheet { + signer: Arc::clone(&signer), + })) + .with_media_signer(Arc::new(EncypherMedia::new(signer))) + .build(), + )) +} + +/// The middleware, which signs the article and the embedded media of each +/// admitted page. +pub struct Encypher { + signer: Arc, +} + +impl std::fmt::Debug for Encypher { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("Encypher").finish_non_exhaustive() + } +} + +/// What the preparation decided for one document: the page to sign, or none. +type Admission = Option; + +impl Middleware for Encypher { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == HTML_MEDIA_TYPE + } + + fn source_protection(&self) -> Option { + Some(embedded::source_protection) + } + + /// Every path but an API, feed, admin or well-known path, or an asset, + /// which is never an article. + fn handles_path(&self, path: &str) -> bool { + !is_excluded_path(path) + } + + /// Serve only: the signed page is written on each reader's copy, so a + /// failed signing is never stored for every reader. + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + /// Applies the request gates, with no I/O, and keeps the page to sign. + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + let admission = admit(&call, context, |page| self.signer.admit(page)); + Some(Box::pin(async move { + Ok(Box::new(admission) as Box) + })) + } + + /// Holds the admitted page, with the security policies its response + /// carries, which decide whether inline style media may be edited. + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let Some(page) = context.take_prepared::(MODULE).flatten() else { + return MiddlewareAction::pass(); + }; + let policies = context + .response_headers + .and_then(embedded::content_security_policies); + MiddlewareAction { + stream: Some(Box::new(document_processor(&self.signer, page, policies))), + ..MiddlewareAction::pass() + } + } +} + +/// The stylesheet middleware, which signs the media `data:` URIs of each +/// admitted stylesheet. +pub struct EncypherStylesheet { + signer: Arc, +} + +impl std::fmt::Debug for EncypherStylesheet { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("EncypherStylesheet") + .finish_non_exhaustive() + } +} + +impl Middleware for EncypherStylesheet { + fn middleware_id(&self) -> &'static str { + STYLESHEET_MIDDLEWARE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == STYLESHEET_MEDIA_TYPE + } + + /// Every path but an API, feed, admin or well-known path. A `.css` + /// suffix is what a stylesheet has. + fn handles_path(&self, path: &str) -> bool { + !is_excluded_prefix(path) + } + + /// Serve only, as for pages. + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + /// Applies the request gates, with no I/O, and keeps the stylesheet to + /// sign. + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + let admission = admit(&call, context, |page| self.signer.admit_stylesheet(page)); + Some(Box::pin(async move { + Ok(Box::new(admission) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let Some(page) = context + .take_prepared::(STYLESHEET_MIDDLEWARE) + .flatten() + else { + return MiddlewareAction::pass(); + }; + MiddlewareAction { + stream: Some(Box::new(stylesheet_processor(&self.signer, page))), + ..MiddlewareAction::pass() + } + } +} + +/// The document a request is for, when `gate` admits it. +fn admit( + call: &ModuleCall<'_>, + context: &MiddlewareContext<'_>, + gate: impl FnOnce(PageRequest) -> Result, +) -> Admission { + let request = ModuleRequest::from_module_context(call).ok()?; + let removed_prefix = request.removed_prefix().unwrap_or_default(); + let page = PageRequest { + method: request.method().clone(), + // The address the reader asked for: a host that serves several + // publishers by path signs each page for its own address. + page_url: format!( + "{}://{}{removed_prefix}{}", + request.scheme(), + request.host(), + request.path() + ), + removed_prefix: removed_prefix.to_owned(), + response_status: context.status.as_u16(), + origin_response_is_private: context.origin_private, + }; + match gate(page) { + Ok(page) => Some(page), + Err(gate) => { + log::debug!("encypher: document not held, gate={}", gate.code()); + None + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/provenance/encypher/src/media_api.rs b/crates/provenance/encypher/src/media_api.rs new file mode 100644 index 0000000000..2d8d7de9aa --- /dev/null +++ b/crates/provenance/encypher/src/media_api.rs @@ -0,0 +1,831 @@ +//! Wire contract of the Encypher media signing endpoint, +//! `POST /api/v1/public/cdn/media/sign`. +//! +//! The request is `multipart/form-data` with a `metadata` JSON part and the +//! exact `source` bytes. Neither core nor this module transforms media, so no +//! `final` part is sent and the execution record lists no operations. A `200` +//! answer is the exact signed bytes, described by `X-Encypher-*` headers that +//! [`verify_signed_response`] checks before any byte reaches a reader. + +use bytes::{BufMut as _, Bytes, BytesMut}; +use http::{HeaderMap, HeaderValue}; +use serde::ser::SerializeSeq as _; +use serde::{Deserialize, Serialize, Serializer}; +use sha2::{Digest as _, Sha256}; +use trusted_server_core::middleware::media_type_of; + +use crate::api::valid_policy_digest; + +/// The media signing route, below the API origin. +pub(crate) const MEDIA_SIGN_PATH: &str = "/api/v1/public/cdn/media/sign"; +/// The request header carrying the replay key the endpoint requires. +pub(crate) const IDEMPOTENCY_KEY_HEADER: &str = "idempotency-key"; +/// The whole multipart body the endpoint accepts. +pub(crate) const MAX_REQUEST_BYTES: usize = 100_000_000; +/// What signing may add to the bytes: the manifest store with its +/// certificate chain and the embedded source data. +pub(crate) const MAX_MANIFEST_OVERHEAD_BYTES: usize = 10 * 1024 * 1024; +/// A failure body is only read for its code. +pub(crate) const MAX_ERROR_BODY_BYTES: usize = 4096; +/// Longest manifest instance or document identifier accepted. +const MAX_IDENTIFIER_BYTES: usize = 256; +/// Longest URL the endpoint accepts or returns. +const MAX_URL_BYTES: usize = 2048; +/// Longest failure code kept for the log. +const MAX_ERROR_CODE_BYTES: usize = 64; + +const POLICY_DIGEST_HEADER: &str = "x-encypher-policy-digest"; +const SIGNED_SHA256_HEADER: &str = "x-encypher-signed-sha256"; +const INSTANCE_ID_HEADER: &str = "x-encypher-manifest-instance-id"; +const DISPOSITION_HEADER: &str = "x-encypher-manifest-disposition"; +const MANIFEST_URL_HEADER: &str = "x-encypher-manifest-url"; +const MANIFEST_SHA256_HEADER: &str = "x-encypher-manifest-sha256"; +const LINK_REQUIRED_HEADER: &str = "x-encypher-manifest-link-required"; +const VERIFICATION_URL_HEADER: &str = "x-encypher-verification-url"; +const DOCUMENT_ID_HEADER: &str = "x-encypher-document-id"; + +/// Every signing header the endpoint answers with, in the order a cached +/// answer stores them. +pub(crate) const SIGNING_HEADERS: [&str; 10] = [ + "content-type", + POLICY_DIGEST_HEADER, + SIGNED_SHA256_HEADER, + INSTANCE_ID_HEADER, + DISPOSITION_HEADER, + MANIFEST_URL_HEADER, + MANIFEST_SHA256_HEADER, + LINK_REQUIRED_HEADER, + VERIFICATION_URL_HEADER, + DOCUMENT_ID_HEADER, +]; + +/// The `metadata` part of one media signing request. +#[derive(Debug, Serialize)] +pub(crate) struct MediaMetadata<'a> { + /// The reader-facing absolute URL the media is delivered on or in. + pub(crate) page_url: &'a str, + /// The server policy the edge holds, which the endpoint refuses when + /// stale. + pub(crate) expected_policy_digest: &'a str, + pub(crate) source: SourceMetadata<'a>, + pub(crate) execution_record: ExecutionRecord<'a>, + pub(crate) delivery: Delivery, +} + +/// What the edge says about the captured source bytes. +#[derive(Debug, Serialize)] +pub(crate) struct SourceMetadata<'a> { + /// A canonical `https` address of the source, or `null` where there is + /// none: embedded media, or an address with a query or credentials, + /// which is never stripped to make one. Without it the endpoint binds the + /// source by embedding its bytes. + pub(crate) reference: Option<&'a str>, + /// The exact hostname the source was fetched from, which the endpoint + /// holds the reference and the claim's managed origins against, or + /// `null` only for media embedded in a document, which was fetched from + /// no host of its own. Never a scheme, port, path or query. + pub(crate) origin_host: Option<&'a str>, + /// The declared media type, a hint the endpoint checks against the bytes. + pub(crate) mime: &'a str, + pub(crate) sha256: &'a str, + pub(crate) byte_length: usize, + /// The source is a platform image optimizer's answer: the endpoint + /// refuses it unless the bytes carry an embedded manifest, which it + /// validates as it validates any ingredient's. Sent only when true. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub(crate) require_embedded_manifest: bool, +} + +/// What the edge did to the source before it was delivered: nothing. +#[derive(Debug, Serialize)] +pub(crate) struct ExecutionRecord<'a> { + pub(crate) source_sha256: &'a str, + pub(crate) final_sha256: &'a str, + /// The exact source was captured and no operation was applied. + pub(crate) complete: bool, + pub(crate) operations: NoOperations, +} + +/// The empty operation list of media delivered as it was captured. +#[derive(Debug)] +pub(crate) struct NoOperations; + +impl Serialize for NoOperations { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_seq(Some(0))?.end() + } +} + +/// How the signed representation reaches readers. +#[derive(Debug, Serialize)] +pub(crate) struct Delivery { + /// Whether every response of the representation can carry a `Link` + /// header naming a remote manifest. + pub(crate) http_link_header: bool, +} + +impl<'a> MediaMetadata<'a> { + /// Metadata for `source` delivered unmodified: no `final` part, an empty + /// and complete execution record. + pub(crate) fn unmodified( + page_url: &'a str, + expected_policy_digest: &'a str, + source: SourceMetadata<'a>, + http_link_header: bool, + ) -> Self { + let sha256 = source.sha256; + Self { + page_url, + expected_policy_digest, + source, + execution_record: ExecutionRecord { + source_sha256: sha256, + final_sha256: sha256, + complete: true, + operations: NoOperations, + }, + delivery: Delivery { http_link_header }, + } + } +} + +/// A built multipart request body and its `Content-Type`. +#[derive(Debug)] +pub(crate) struct Multipart { + pub(crate) content_type: String, + pub(crate) body: Bytes, +} + +/// Builds the `multipart/form-data` body with the `metadata` and `source` +/// parts, with a boundary that occurs in neither. +pub(crate) fn multipart(metadata: &[u8], source: &[u8]) -> Multipart { + let seed = sha256_hex(metadata); + let mut attempt = 0_u32; + let boundary = loop { + let candidate = format!( + "encypher-media-{}-{attempt}", + seed.get(..32).unwrap_or(&seed) + ); + let delimiter = format!("--{candidate}"); + if !contains(metadata, delimiter.as_bytes()) && !contains(source, delimiter.as_bytes()) { + break candidate; + } + attempt = attempt.saturating_add(1); + }; + let metadata_head = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"metadata\"\r\n\ + Content-Type: application/json\r\n\r\n" + ); + let source_head = format!( + "\r\n--{boundary}\r\nContent-Disposition: form-data; name=\"source\"; \ + filename=\"source\"\r\nContent-Type: application/octet-stream\r\n\r\n" + ); + let closing = format!("\r\n--{boundary}--\r\n"); + let mut body = BytesMut::with_capacity( + metadata_head.len() + metadata.len() + source_head.len() + source.len() + closing.len(), + ); + body.put_slice(metadata_head.as_bytes()); + body.put_slice(metadata); + body.put_slice(source_head.as_bytes()); + body.put_slice(source); + body.put_slice(closing.as_bytes()); + Multipart { + content_type: format!("multipart/form-data; boundary={boundary}"), + body: body.freeze(), + } +} + +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +/// Lowercase hex SHA-256 of `bytes`. +pub(crate) fn sha256_hex(bytes: &[u8]) -> String { + hex_lower(&Sha256::digest(bytes)) +} + +fn hex_lower(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(char::from(DIGITS[usize::from(byte >> 4)])); + out.push(char::from(DIGITS[usize::from(byte & 0x0f)])); + } + out +} + +/// The bare media type an answer must carry for media declared as `declared`, +/// with the common `image/jpg` spelling read as `image/jpeg`, and the +/// `audio/x-aac` and `audio/aacp` spellings of raw AAC as `audio/aac`. +pub(crate) fn canonical_media_type(declared: &str) -> String { + let media_type = media_type_of(declared); + match media_type.as_str() { + "image/jpg" => "image/jpeg".to_owned(), + "audio/x-aac" | "audio/aacp" => "audio/aac".to_owned(), + _ => media_type, + } +} + +/// Whether `reference` may be sent as the source's address: canonical +/// `https` on the default port, with a path, no credentials, query or +/// fragment, on `host`. Anything else is sent as no reference. +pub(crate) fn safe_source_reference(reference: &str, host: &str) -> bool { + if reference.len() > MAX_URL_BYTES || !reference.is_ascii() { + return false; + } + let Ok(parsed) = url::Url::parse(reference) else { + return false; + }; + parsed.scheme() == "https" + && parsed.as_str() == reference + && parsed.username().is_empty() + && parsed.password().is_none() + && parsed.port().is_none() + && parsed.query().is_none() + && parsed.fragment().is_none() + && parsed.path() != "/" + && parsed.host_str() == Some(host) +} + +/// Whether `host` is one canonical hostname alone: no scheme, user +/// information, port, path or query, in lower case. +pub(crate) fn valid_origin_host(host: &str) -> bool { + !host.is_empty() + && host.len() <= 253 + && url::Url::parse(&format!("https://{host}/")).is_ok_and(|parsed| { + parsed.host_str() == Some(host) + && parsed.port().is_none() + && parsed.username().is_empty() + && parsed.path() == "/" + }) +} + +/// Where the signed bytes' manifest lives. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Disposition { + /// Inside the signed bytes. + Embedded, + /// At [`SignedMedia::manifest_url`], named by the bytes or a `Link`. + External, +} + +/// What a signing answer must match. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Expected<'a> { + pub(crate) policy_digest: &'a str, + /// The bare media type the source was delivered as. + pub(crate) media_type: &'a str, + /// SHA-256 of the bytes as they were before signing. + pub(crate) unsigned_sha256: &'a str, + /// Whether every response of the representation can carry a `Link`. + pub(crate) link_deliverable: bool, +} + +/// A verified signing answer. +#[derive(Debug, Clone)] +pub(crate) struct SignedMedia { + pub(crate) bytes: Bytes, + pub(crate) content_type: HeaderValue, + pub(crate) instance_id: String, + pub(crate) document_id: String, + pub(crate) verification_url: String, + pub(crate) disposition: Disposition, + pub(crate) manifest_url: Option, +} + +impl SignedMedia { + /// The `Link` header value naming a remote manifest, when there is one. + pub(crate) fn link(&self) -> Option { + self.manifest_url + .as_deref() + .map(|url| format!("<{url}>; rel=\"c2pa-manifest\"")) + } +} + +/// Why a `200` answer is not accepted as signed media. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ResponseError { + /// The answer's media type is not the one the source was delivered as. + ContentType, + /// The answer was signed under another policy. + PolicyDigest, + /// The body is not the bytes the signed digest names. + SignedDigest, + /// A header is missing, repeated or malformed. + Malformed, + /// The manifest is remote and needs a `Link` the delivery cannot carry. + LinkUndeliverable, + /// The body carries no manifest and no reference to one. + Unsigned, +} + +/// The one value of `name`, when it appears exactly once as visible ASCII. +fn single<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + let mut values = headers.get_all(name).iter(); + let value = values.next()?; + if values.next().is_some() { + return None; + } + value.to_str().ok() +} + +fn valid_identifier(value: &str) -> bool { + !value.is_empty() + && value.len() <= MAX_IDENTIFIER_BYTES + && value.bytes().all(|byte| byte.is_ascii_graphic()) +} + +/// Whether `value` is a canonical absolute `https` URL that is safe inside a +/// `Link` header's angle brackets. +fn valid_https_url(value: &str) -> bool { + value.len() <= MAX_URL_BYTES + && value + .bytes() + .all(|byte| byte.is_ascii_graphic() && !matches!(byte, b'<' | b'>' | b'"')) + && url::Url::parse(value).is_ok_and(|parsed| { + parsed.scheme() == "https" && parsed.host_str().is_some() && parsed.as_str() == value + }) +} + +/// Checks a `200` signing answer against what was sent, and keeps it only +/// when it is signed media this delivery can carry. +/// +/// # Errors +/// +/// Returns the first [`ResponseError`] the answer shows. +pub(crate) fn verify_signed_response( + headers: &HeaderMap, + body: Bytes, + expected: &Expected<'_>, +) -> Result { + let content_type = single(headers, "content-type").ok_or(ResponseError::Malformed)?; + if canonical_media_type(content_type) != expected.media_type { + return Err(ResponseError::ContentType); + } + let policy_digest = single(headers, POLICY_DIGEST_HEADER).ok_or(ResponseError::Malformed)?; + if !valid_policy_digest(policy_digest) { + return Err(ResponseError::Malformed); + } + if policy_digest != expected.policy_digest { + return Err(ResponseError::PolicyDigest); + } + let signed_sha256 = single(headers, SIGNED_SHA256_HEADER).ok_or(ResponseError::Malformed)?; + if !valid_policy_digest(signed_sha256) { + return Err(ResponseError::Malformed); + } + if body.is_empty() || sha256_hex(&body) != signed_sha256 { + return Err(ResponseError::SignedDigest); + } + let instance_id = single(headers, INSTANCE_ID_HEADER) + .filter(|value| valid_identifier(value)) + .ok_or(ResponseError::Malformed)?; + let document_id = single(headers, DOCUMENT_ID_HEADER) + .filter(|value| valid_identifier(value)) + .ok_or(ResponseError::Malformed)?; + let verification_url = single(headers, VERIFICATION_URL_HEADER) + .filter(|value| valid_https_url(value)) + .ok_or(ResponseError::Malformed)?; + let link_required = match single(headers, LINK_REQUIRED_HEADER) { + Some("true") => true, + Some("false") => false, + _ => return Err(ResponseError::Malformed), + }; + let manifest_url = headers.get(MANIFEST_URL_HEADER).map(|_| { + single(headers, MANIFEST_URL_HEADER) + .filter(|value| valid_https_url(value)) + .ok_or(ResponseError::Malformed) + }); + let changed = signed_sha256 != expected.unsigned_sha256; + let disposition = match single(headers, DISPOSITION_HEADER) { + Some("embedded") => { + // The manifest store is in the bytes, so they changed, hold the + // `c2pa` label and point nowhere else. These are consistency + // checks against a wrong answer, not a signature check: the + // signature is trusted through the authenticated endpoint and + // the digest binding above. + if manifest_url.is_some() || link_required { + return Err(ResponseError::Malformed); + } + if !changed || !contains(&body, b"c2pa") { + return Err(ResponseError::Unsigned); + } + Disposition::Embedded + } + Some("external") => { + if manifest_url.is_none() { + return Err(ResponseError::Malformed); + } + single(headers, MANIFEST_SHA256_HEADER) + .filter(|value| valid_policy_digest(value)) + .ok_or(ResponseError::Malformed)?; + if link_required && !expected.link_deliverable { + return Err(ResponseError::LinkUndeliverable); + } + // A remote manifest no `Link` names must be named by the bytes. + if !link_required && !changed { + return Err(ResponseError::Unsigned); + } + Disposition::External + } + _ => return Err(ResponseError::Malformed), + }; + let manifest_url = manifest_url.transpose()?.map(str::to_owned); + let content_type = HeaderValue::from_str(content_type).map_err(|_| ResponseError::Malformed)?; + Ok(SignedMedia { + bytes: body, + content_type, + instance_id: instance_id.to_owned(), + document_id: document_id.to_owned(), + verification_url: verification_url.to_owned(), + disposition, + manifest_url, + }) +} + +/// The failure code of an error answer, kept only as a short identifier. +pub(crate) fn error_code(body: &[u8]) -> Option { + #[derive(Deserialize)] + struct ErrorBody { + error: ErrorDetail, + } + #[derive(Deserialize)] + struct ErrorDetail { + code: String, + } + if body.len() > MAX_ERROR_BODY_BYTES { + return None; + } + let parsed: ErrorBody = serde_json::from_slice(body).ok()?; + let code = parsed.error.code; + (!code.is_empty() + && code.len() <= MAX_ERROR_CODE_BYTES + && code + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')) + .then_some(code) +} + +#[cfg(test)] +mod tests { + use http::HeaderName; + use serde_json::json; + + use super::*; + + const DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const UNSIGNED: &[u8] = b"\x89PNG\r\n\x1a\nunsigned"; + const SIGNED: &[u8] = b"\x89PNG\r\n\x1a\nunsigned-caBXjumbc2pa-manifest"; + + fn expected(unsigned_sha256: &str, link_deliverable: bool) -> Expected<'_> { + Expected { + policy_digest: DIGEST, + media_type: "image/png", + unsigned_sha256, + link_deliverable, + } + } + + fn embedded_headers(body: &[u8]) -> HeaderMap { + let mut headers = HeaderMap::new(); + for (name, value) in [ + ("content-type", "image/png".to_owned()), + (POLICY_DIGEST_HEADER, DIGEST.to_owned()), + (SIGNED_SHA256_HEADER, sha256_hex(body)), + (INSTANCE_ID_HEADER, "xmp:iid:0001".to_owned()), + (DISPOSITION_HEADER, "embedded".to_owned()), + (LINK_REQUIRED_HEADER, "false".to_owned()), + ( + VERIFICATION_URL_HEADER, + "https://verify.example.com/doc-1".to_owned(), + ), + (DOCUMENT_ID_HEADER, "doc-1".to_owned()), + ] { + headers.insert( + HeaderName::from_static(name), + HeaderValue::from_str(&value).expect("should build header"), + ); + } + headers + } + + fn external_headers(body: &[u8], link_required: bool) -> HeaderMap { + let mut headers = embedded_headers(body); + headers.insert(DISPOSITION_HEADER, HeaderValue::from_static("external")); + headers.insert( + MANIFEST_URL_HEADER, + HeaderValue::from_static("https://api.example.com/manifests/doc-1"), + ); + headers.insert( + MANIFEST_SHA256_HEADER, + HeaderValue::from_str(DIGEST).expect("should build header"), + ); + headers.insert( + LINK_REQUIRED_HEADER, + HeaderValue::from_static(if link_required { "true" } else { "false" }), + ); + headers + } + + #[test] + fn metadata_matches_the_frozen_wire_shape() { + let sha = sha256_hex(UNSIGNED); + let metadata = MediaMetadata::unmodified( + "https://publisher.example.com/news/story", + DIGEST, + SourceMetadata { + reference: None, + origin_host: None, + mime: "image/png", + sha256: &sha, + byte_length: UNSIGNED.len(), + require_embedded_manifest: false, + }, + false, + ); + let value = serde_json::to_value(&metadata).expect("should serialize metadata"); + assert_eq!( + value, + json!({ + "page_url": "https://publisher.example.com/news/story", + "expected_policy_digest": DIGEST, + "source": { + "reference": null, + "origin_host": null, + "mime": "image/png", + "sha256": sha, + "byte_length": UNSIGNED.len(), + }, + "execution_record": { + "source_sha256": sha, + "final_sha256": sha, + "complete": true, + "operations": [], + }, + "delivery": { "http_link_header": false }, + }), + "should send an unmodified, complete record with a null reference" + ); + } + + #[test] + fn an_optimizer_answer_requires_its_embedded_manifest_on_the_wire() { + let sha = sha256_hex(SIGNED); + let metadata = MediaMetadata::unmodified( + "https://publisher.example.com/images/photo.jpg", + DIGEST, + SourceMetadata { + reference: None, + origin_host: Some("origin.example.com"), + mime: "image/png", + sha256: &sha, + byte_length: SIGNED.len(), + require_embedded_manifest: true, + }, + true, + ); + let value = serde_json::to_value(&metadata).expect("should serialize metadata"); + assert_eq!( + value["source"], + json!({ + "reference": null, + "origin_host": "origin.example.com", + "mime": "image/png", + "sha256": sha, + "byte_length": SIGNED.len(), + "require_embedded_manifest": true, + }), + "should require the optimizer's embedded manifest and name no address" + ); + assert_eq!( + value["execution_record"], + json!({ + "source_sha256": sha, + "final_sha256": sha, + "complete": true, + "operations": [], + }), + "should report no operation of the edge's own" + ); + } + + #[test] + fn multipart_carries_exact_source_bytes_under_an_absent_boundary() { + let source = b"--encypher-media-binary\r\n\x00\xff"; + let built = multipart(br#"{"page_url":"x"}"#, source); + let boundary = built + .content_type + .strip_prefix("multipart/form-data; boundary=") + .expect("should declare a boundary"); + let body = built.body.as_ref(); + let source_start = body + .windows(4) + .position(|window| window == b"\r\n\r\n") + .and_then(|first| { + body[first + 4..] + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|second| first + 4 + second + 4) + }) + .expect("should find the source part body"); + let tail = format!("\r\n--{boundary}--\r\n"); + assert!(body.ends_with(tail.as_bytes()), "should close the body"); + assert_eq!( + &body[source_start..body.len() - tail.len()], + source, + "should carry the exact source bytes" + ); + assert!( + body.windows(9).any(|window| window == b"name=\"sou"), + "should name the source part" + ); + assert!( + !body.windows(7).any(|window| window == b"\"final\""), + "should send no final part for unmodified media" + ); + } + + #[test] + fn an_embedded_answer_is_accepted_only_with_matching_bytes_and_policy() { + let unsigned = sha256_hex(UNSIGNED); + let signed = verify_signed_response( + &embedded_headers(SIGNED), + Bytes::from_static(SIGNED), + &expected(&unsigned, false), + ) + .expect("should accept a verified embedded answer"); + assert_eq!(signed.bytes.as_ref(), SIGNED, "should keep the exact bytes"); + assert_eq!( + signed.disposition, + Disposition::Embedded, + "should be embedded" + ); + assert_eq!(signed.link(), None, "should need no Link"); + + let tampered = verify_signed_response( + &embedded_headers(SIGNED), + Bytes::from_static(b"\x89PNG\r\n\x1a\nc2pa-other"), + &expected(&unsigned, false), + ); + assert_eq!( + tampered.expect_err("should refuse a body the digest does not name"), + ResponseError::SignedDigest + ); + + let mut stale = embedded_headers(SIGNED); + stale.insert( + POLICY_DIGEST_HEADER, + HeaderValue::from_str(&"b".repeat(64)).expect("should build header"), + ); + assert_eq!( + verify_signed_response( + &stale, + Bytes::from_static(SIGNED), + &expected(&unsigned, false) + ) + .expect_err("should refuse another policy"), + ResponseError::PolicyDigest + ); + + let mut retyped = embedded_headers(SIGNED); + retyped.insert("content-type", HeaderValue::from_static("image/jpeg")); + assert_eq!( + verify_signed_response( + &retyped, + Bytes::from_static(SIGNED), + &expected(&unsigned, false) + ) + .expect_err("should refuse a changed media type"), + ResponseError::ContentType + ); + } + + #[test] + fn unchanged_bytes_are_never_accepted_as_signed() { + let unsigned = sha256_hex(UNSIGNED); + assert_eq!( + verify_signed_response( + &embedded_headers(UNSIGNED), + Bytes::from_static(UNSIGNED), + &expected(&unsigned, false), + ) + .expect_err("should refuse an unsigned embedded answer"), + ResponseError::Unsigned + ); + assert_eq!( + verify_signed_response( + &external_headers(UNSIGNED, false), + Bytes::from_static(UNSIGNED), + &expected(&unsigned, true), + ) + .expect_err("should refuse a remote manifest nothing names"), + ResponseError::Unsigned + ); + } + + #[test] + fn a_remote_manifest_needs_a_deliverable_link() { + let unsigned = sha256_hex(UNSIGNED); + assert_eq!( + verify_signed_response( + &external_headers(UNSIGNED, true), + Bytes::from_static(UNSIGNED), + &expected(&unsigned, false), + ) + .expect_err("should refuse a Link the delivery cannot carry"), + ResponseError::LinkUndeliverable + ); + let signed = verify_signed_response( + &external_headers(UNSIGNED, true), + Bytes::from_static(UNSIGNED), + &expected(&unsigned, true), + ) + .expect("should accept a remote manifest the delivery links"); + assert_eq!( + signed.link().as_deref(), + Some("; rel=\"c2pa-manifest\""), + "should name the manifest in a Link" + ); + } + + #[test] + fn repeated_or_malformed_signing_headers_are_refused() { + let unsigned = sha256_hex(UNSIGNED); + let mut repeated = embedded_headers(SIGNED); + repeated.append(INSTANCE_ID_HEADER, HeaderValue::from_static("xmp:iid:0002")); + let mut bad_url = embedded_headers(SIGNED); + bad_url.insert( + VERIFICATION_URL_HEADER, + HeaderValue::from_static("http://verify.example.com/doc-1"), + ); + let mut bad_disposition = embedded_headers(SIGNED); + bad_disposition.insert(DISPOSITION_HEADER, HeaderValue::from_static("sidecar")); + for headers in [repeated, bad_url, bad_disposition] { + assert_eq!( + verify_signed_response( + &headers, + Bytes::from_static(SIGNED), + &expected(&unsigned, false) + ) + .expect_err("should refuse malformed headers"), + ResponseError::Malformed + ); + } + } + + #[test] + fn common_misspellings_name_the_media_type_the_service_lists() { + for (declared, canonical) in [ + ("image/jpg", "image/jpeg"), + ("audio/x-aac", "audio/aac"), + ("Audio/AACP; codecs=aac", "audio/aac"), + ("audio/aac", "audio/aac"), + ("audio/mp4", "audio/mp4"), + ] { + assert_eq!(canonical_media_type(declared), canonical, "{declared}"); + } + } + + #[test] + fn source_references_are_sent_only_when_canonical_and_on_the_claim_host() { + let host = "publisher.example.com"; + assert!( + safe_source_reference("https://publisher.example.com/images/a.png", host), + "should send a canonical https address" + ); + for unsafe_reference in [ + "https://publisher.example.com/images/a.png?token=secret", + "https://user:pass@publisher.example.com/images/a.png", + "https://publisher.example.com/images/a.png#frag", + "http://publisher.example.com/images/a.png", + "https://publisher.example.com:8443/images/a.png", + "https://origin.example.com/images/a.png", + "https://PUBLISHER.example.com/images/a.png", + "https://publisher.example.com/", + ] { + assert!( + !safe_source_reference(unsafe_reference, host), + "should never send {unsafe_reference}" + ); + } + } + + #[test] + fn error_codes_are_read_as_short_identifiers_only() { + let body = serde_json::to_vec(&json!({ + "success": false, + "error": {"code": "TRUSTED_SERVER_POLICY_CHANGED", "message": "stale"}, + "correlation_id": "c-1", + })) + .expect("should serialize"); + assert_eq!( + error_code(&body).as_deref(), + Some("TRUSTED_SERVER_POLICY_CHANGED") + ); + let injected = + serde_json::to_vec(&json!({"error": {"code": "a b\nc"}})).expect("should serialize"); + assert_eq!(error_code(&injected), None, "should drop an unsafe code"); + } +} diff --git a/crates/provenance/encypher/src/prepared_probe.rs b/crates/provenance/encypher/src/prepared_probe.rs new file mode 100644 index 0000000000..2521442616 --- /dev/null +++ b/crates/provenance/encypher/src/prepared_probe.rs @@ -0,0 +1,995 @@ +//! The signed probe release, run on the edge over an asynchronous source. +//! +//! Large media is signed in prepared-hash mode: Encypher's planner decides +//! which structure bytes a manifest binds and how the carrier is built, and +//! the edge never holds the whole file. The planner's format logic ships only +//! inside Encypher's signed probe release, a WebAssembly artifact the signing +//! service advertises and serves. This module is the generic host for that +//! artifact. It holds no format, planner or hashing algorithm of its own: it +//! verifies a release, runs the operations the release implements (`extract`, +//! `verify_plan`, `build_carrier`, `binding_items`, `check_store` and the +//! rest), and returns the JSON the release answers with. +//! +//! # Release trust +//! +//! A release is run only after every check the advertised release contract +//! names, in order: the descriptor's Ed25519 signature (`verify_strict` over +//! its canonical bytes) by a key this build pins, a release that is not +//! revoked, ABI 1, a release at or above the floor, an artifact of at most +//! 4 MiB whose SHA-256 is the descriptor's, a module that imports exactly +//! `env.host_read` and exports the probe ABI, a `probe_abi` of 1, and a +//! `version` that reports the descriptor's `planner_id`. The caller supplies +//! the artifact bytes, fetched through the runtime's own HTTP client and +//! caches from [`ProbeDescriptor::artifact_path`]; nothing here touches the +//! network. +//! +//! # Asynchronous reads +//! +//! The release's only way to see bytes is its one import, +//! `host_read(offset, len, dst) -> status`, which the release calls +//! synchronously. On the edge a read is an HTTP range request, so it cannot +//! complete inside the interpreter's callback without blocking the executor. +//! The callback therefore checks the request against the source size and the +//! per-read bound and, when it is admissible, stops the interpreter with a +//! resumable host trap. [`ProbeModule::call`] then awaits the source outside +//! the interpreter, writing straight into the release's own read buffer in +//! linear memory, and resumes the suspended call with the status. The +//! operation is never restarted, so each read happens once, in order, and no +//! host-side copy of the file exists. +//! +//! Each call runs in a fresh instance with its own fuel, memory, per-read, +//! total-read and result bounds, so an abandoned call leaves nothing behind +//! and one [`ProbeModule`] serves concurrent calls. + +use std::convert::Infallible; +use std::fmt; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use ed25519_dalek::{Signature, VerifyingKey}; +use serde_json::{Value, json}; +use wasmi::{ + Caller, Config, Engine, ExternType, Linker, Memory, Module, Store, StoreLimits, + StoreLimitsBuilder, TrapCode, TypedFunc, TypedResumableCall, Val, ValType, +}; + +use crate::media_api::sha256_hex; + +/// The probe host ABI this host implements. +pub(crate) const PROBE_ABI: u64 = 1; + +/// Encypher's production probe release key: a raw Ed25519 public key, lower +/// case hex `9ae0bf4a...6b75d9`. Pinning it means a release Encypher did not +/// sign is refused even when the API or a cache serving the artifact is +/// compromised. +const PRODUCTION_RELEASE_KEY: [u8; 32] = [ + 0x9a, 0xe0, 0xbf, 0x4a, 0xea, 0x51, 0x40, 0x00, 0x8d, 0x2f, 0x48, 0xe6, 0xc0, 0x86, 0x91, 0x36, + 0x0a, 0x7f, 0xe2, 0x3e, 0x0d, 0x36, 0xf2, 0x45, 0x22, 0xc7, 0x70, 0xb7, 0x64, 0x6b, 0x75, 0xd9, +]; + +/// The lowest probe release this build runs. Raise it when Encypher +/// withdraws a release, so a compromised API cannot downgrade the edge to it. +const PRODUCTION_RELEASE_FLOOR: u64 = 8; + +/// A release artifact is about 800 KB; anything over this is not one. +const MAX_ARTIFACT_BYTES: usize = 4 * 1024 * 1024; + +/// `host_read` succeeded and the destination holds the requested bytes. +const READ_OK: i32 = 0; +/// `host_read` named a destination outside the release's linear memory. +const READ_MEMORY_OUT_OF_BOUNDS: i32 = 4; +/// `host_read` asked for more than one read may carry or for bytes past the +/// end of the source. +const READ_OUT_OF_BOUNDS: i32 = 5; + +/// A signed release descriptor, as the capabilities response and a 409 +/// refusal carry it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ProbeDescriptor { + pub(crate) abi: u64, + pub(crate) planner_id: String, + pub(crate) release: u64, + pub(crate) sha256: String, + pub(crate) signature: String, +} + +impl ProbeDescriptor { + /// Reads one descriptor, or `None` when its shape is not the contract's: + /// unsigned integers `abi` and `release`, lower case 64-hex `planner_id` + /// and `sha256`, and a non-empty `signature`. + #[must_use] + pub(crate) fn parse(value: &Value) -> Option { + let hex64 = |text: &&str| { + text.len() == 64 + && text + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) + }; + Some(Self { + abi: value.get("abi")?.as_u64()?, + planner_id: value.get("planner_id")?.as_str().filter(hex64)?.to_owned(), + release: value.get("release")?.as_u64()?, + sha256: value.get("sha256")?.as_str().filter(hex64)?.to_owned(), + signature: value + .get("signature")? + .as_str() + .filter(|text| !text.is_empty())? + .to_owned(), + }) + } + + /// Every well-formed descriptor in a `probe` array; malformed entries + /// are dropped, and anything other than an array is none. + #[must_use] + pub(crate) fn parse_list(value: Option<&Value>) -> Vec { + value + .and_then(Value::as_array) + .map(|items| items.iter().filter_map(Self::parse).collect()) + .unwrap_or_default() + } + + #[cfg(test)] + /// The descriptor as the signing API receives it back. + #[must_use] + pub(crate) fn to_json(&self) -> Value { + json!({ + "abi": self.abi, + "planner_id": self.planner_id, + "release": self.release, + "sha256": self.sha256, + "signature": self.signature, + }) + } + + /// The exact bytes the release key signs: the four signed fields, keys + /// sorted, no whitespace. + #[must_use] + pub(crate) fn canonical_json(&self) -> String { + format!( + "{{\"abi\":{},\"planner_id\":\"{}\",\"release\":{},\"sha256\":\"{}\"}}", + self.abi, self.planner_id, self.release, self.sha256 + ) + } + + /// The signing API path that serves this release's artifact, relative to + /// the API base. + #[must_use] + pub(crate) fn artifact_path(&self) -> String { + format!("sign/media/probe/{}.wasm", self.sha256) + } +} + +/// The release keys, floor and revocations a release must satisfy. +/// +/// Production code obtains only [`ReleaseAuthority::production`]: the keys +/// are compiled in and nothing a configuration or a request carries can +/// change them. +#[derive(Debug, Clone)] +pub(crate) struct ReleaseAuthority { + keys: Vec, + floor: u64, + /// Lower case artifact SHA-256s withdrawn for cause. + revoked: Vec, +} + +impl ReleaseAuthority { + /// Encypher's production release key and this build's floor. + /// + /// # Panics + /// + /// Never: the pinned key is a valid Ed25519 point, which a test proves. + #[must_use] + pub(crate) fn production() -> Self { + let key = VerifyingKey::from_bytes(&PRODUCTION_RELEASE_KEY) + .expect("should decode the pinned production release key"); + Self { + keys: vec![key], + floor: PRODUCTION_RELEASE_FLOOR, + revoked: Vec::new(), + } + } + + /// An authority over test keys. + #[cfg(test)] + pub(crate) fn pinned(keys: Vec, floor: u64, revoked: Vec) -> Self { + Self { + keys, + floor, + revoked, + } + } + + /// Checks a descriptor before any of its release's bytes are fetched: + /// revocation, signature, ABI and floor. + /// + /// # Errors + /// + /// The [`ReleaseError`] naming the first check the descriptor fails. + pub(crate) fn verify(&self, descriptor: &ProbeDescriptor) -> Result<(), ReleaseError> { + if self + .revoked + .iter() + .any(|sha256| sha256.eq_ignore_ascii_case(&descriptor.sha256)) + { + return Err(ReleaseError::Revoked { + release: descriptor.release, + }); + } + let signature = STANDARD + .decode(&descriptor.signature) + .ok() + .and_then(|bytes| Signature::from_slice(&bytes).ok()) + .ok_or(ReleaseError::MalformedSignature { + release: descriptor.release, + })?; + let message = descriptor.canonical_json(); + let signed = self + .keys + .iter() + .any(|key| key.verify_strict(message.as_bytes(), &signature).is_ok()); + if !signed { + return Err(ReleaseError::Unsigned { + release: descriptor.release, + }); + } + if descriptor.abi != PROBE_ABI { + return Err(ReleaseError::AbiUnsupported { + abi: descriptor.abi, + }); + } + if descriptor.release < self.floor { + return Err(ReleaseError::BelowFloor { + release: descriptor.release, + floor: self.floor, + }); + } + Ok(()) + } + + /// The newest advertised release this host implements the ABI of, + /// verified. + /// + /// # Errors + /// + /// [`ReleaseError::NoneAdvertised`] for an empty list, + /// [`ReleaseError::NoCompatibleRelease`] when none uses this ABI, and the + /// chosen descriptor's verification failure otherwise. An older release + /// is never chosen in place of a newer one that fails. + pub(crate) fn select<'a>( + &self, + descriptors: &'a [ProbeDescriptor], + ) -> Result<&'a ProbeDescriptor, ReleaseError> { + if descriptors.is_empty() { + return Err(ReleaseError::NoneAdvertised); + } + let descriptor = descriptors + .iter() + .filter(|descriptor| descriptor.abi == PROBE_ABI) + .max_by_key(|descriptor| descriptor.release) + .ok_or(ReleaseError::NoCompatibleRelease)?; + self.verify(descriptor)?; + Ok(descriptor) + } +} + +/// Why a release was not run. Every variant is refused before the release +/// reads a byte of any source. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ReleaseError { + /// The service advertised no release. + NoneAdvertised, + /// The service advertised no release for [`PROBE_ABI`]. + NoCompatibleRelease, + /// The release was withdrawn for cause. + Revoked { release: u64 }, + /// The signature is not 64 bytes of standard padded base64. + MalformedSignature { release: u64 }, + /// No pinned key signed the descriptor. + Unsigned { release: u64 }, + /// The descriptor names an ABI this host does not implement. + AbiUnsupported { abi: u64 }, + /// The release is older than this build's floor. + BelowFloor { release: u64, floor: u64 }, + /// The artifact is larger than any release. + ArtifactTooLarge { bytes: usize }, + /// The artifact does not hash to the signed digest. + DigestMismatch, + /// The artifact is not a valid WebAssembly module. + InvalidModule(String), + /// The module imports anything other than exactly `env.host_read`. + ImportsMismatch, + /// The module does not export the probe ABI, or its start or + /// `probe_abi` trapped. + ExportsMismatch(String), + /// `probe_abi` reports an ABI other than the descriptor's. + AbiReported { reported: u32 }, + /// `version` failed or did not report the descriptor's `planner_id`. + PlannerMismatch, +} + +impl ReleaseError { + /// The client contract's code for this refusal. + #[must_use] + pub(crate) fn code(&self) -> &'static str { + match self { + Self::NoneAdvertised => "PROBE_RELEASE_UNAVAILABLE", + Self::NoCompatibleRelease | Self::AbiUnsupported { .. } => "PROBE_UPGRADE_REQUIRED", + _ => "PROBE_RELEASE_INVALID", + } + } +} + +impl fmt::Display for ReleaseError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NoneAdvertised => { + formatter.write_str("the signing service advertised no probe release") + } + Self::NoCompatibleRelease => write!( + formatter, + "the signing service advertised no probe release for ABI {PROBE_ABI}" + ), + Self::Revoked { release } => { + write!(formatter, "probe release {release} was withdrawn") + } + Self::MalformedSignature { release } => { + write!( + formatter, + "probe release {release} carries a malformed signature" + ) + } + Self::Unsigned { release } => write!( + formatter, + "probe release {release} is not signed by a pinned release key" + ), + Self::AbiUnsupported { abi } => write!( + formatter, + "the probe release uses ABI {abi}; this host implements ABI {PROBE_ABI}" + ), + Self::BelowFloor { release, floor } => write!( + formatter, + "probe release {release} is below this build's floor of {floor}" + ), + Self::ArtifactTooLarge { bytes } => write!( + formatter, + "the probe artifact is {bytes} bytes, over the {MAX_ARTIFACT_BYTES} byte limit" + ), + Self::DigestMismatch => { + formatter.write_str("the probe artifact does not hash to its signed digest") + } + Self::InvalidModule(error) => write!( + formatter, + "the probe artifact is not a valid WebAssembly module: {error}" + ), + Self::ImportsMismatch => { + formatter.write_str("the probe release imports something other than env.host_read") + } + Self::ExportsMismatch(error) => write!( + formatter, + "the probe release does not implement the host ABI: {error}" + ), + Self::AbiReported { reported } => write!( + formatter, + "the probe release reports ABI {reported}, not its descriptor's" + ), + Self::PlannerMismatch => { + formatter.write_str("the probe release does not report its descriptor's planner") + } + } + } +} + +impl std::error::Error for ReleaseError {} + +/// Bounds on one call. +/// +/// Fuel, the total read and the result abandon the call when exceeded. A +/// read over the per-read bound is answered with status 5, and growth past +/// the memory bound fails inside the release, which then answers as it +/// handles an allocation failure. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ProbeLimits { + /// Linear memory one instance may grow to. + pub(crate) memory_bytes: usize, + /// Instructions one call may execute. + pub(crate) fuel_per_call: u64, + /// The largest single read; a larger request is answered with status 5. + pub(crate) max_read_bytes: u32, + /// Bytes all reads of one call may carry together. + pub(crate) max_total_read_bytes: u64, + /// The largest result a call may return. + pub(crate) max_result_bytes: u32, +} + +impl Default for ProbeLimits { + /// Leave room for the host, parsed JSON and the signed module on + /// 128 MiB-class runtimes. Media payload hashing stays outside WASM. + /// Larger structure or result disclosures are refused explicitly. + fn default() -> Self { + Self { + memory_bytes: 32 * 1024 * 1024, + fuel_per_call: 200_000_000, + max_read_bytes: 8 * 1024 * 1024, + max_total_read_bytes: 64 * 1024 * 1024, + max_result_bytes: 4 * 1024 * 1024, + } + } +} + +/// One version of one file, read at offsets. +/// +/// An implementation MUST serve every read of a call from the same version +/// (an HTTP source pins a strong `ETag` and refuses a range answered from +/// any other) and MUST fail rather than return fewer bytes than asked for. +pub(crate) trait ProbeSource { + /// Why a read failed. The call is abandoned and the error returned as + /// [`ProbeCallError::Source`]. + type Error; + + /// The size of the pinned version, in bytes. The host refuses any read + /// past it before the source is asked. + fn size(&self) -> u64; + + /// Fills `destination` with the bytes at `offset`. + fn read_exact_at( + &self, + offset: u64, + destination: &mut [u8], + ) -> impl Future>; +} + +/// The source of an operation that reads no file, such as `version`, +/// `build_carrier` or `check_store`. It has no bytes, so the host refuses +/// every non-empty read with status 5. +struct NoSource; + +impl ProbeSource for NoSource { + type Error = Infallible; + + fn size(&self) -> u64 { + 0 + } + + async fn read_exact_at(&self, _offset: u64, _destination: &mut [u8]) -> Result<(), Infallible> { + Ok(()) + } +} + +/// The release's own error object, returned with status 1. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ProbeRefusal { + /// The probe's code, such as `PLAN_MISMATCH` or `UNSUPPORTED_FORMAT`. + pub(crate) code: String, + /// Whether the refusal reports an integrity failure. + pub(crate) integrity: bool, + pub(crate) message: String, +} + +/// Why a call ended without the operation's value. +#[derive(Debug)] +pub(crate) enum ProbeCallError { + /// The source failed a read. The suspended call was dropped. + Source(E), + /// The release answered with its error object. + Refused(ProbeRefusal), + /// The host stopped the call, or the release broke the ABI. + Failed(ProbeFailure), +} + +impl ProbeCallError { + /// The client contract's code: the release's own code for a refusal, + /// `PROBE_FAILED` otherwise. + #[must_use] + pub(crate) fn code(&self) -> &str { + match self { + Self::Refused(refusal) => &refusal.code, + Self::Source(_) | Self::Failed(_) => "PROBE_FAILED", + } + } +} + +impl fmt::Display for ProbeCallError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Source(error) => write!(formatter, "the probe's source failed a read: {error}"), + Self::Refused(refusal) => { + write!( + formatter, + "the probe refused: {}: {}", + refusal.code, refusal.message + ) + } + Self::Failed(failure) => failure.fmt(formatter), + } + } +} + +impl std::error::Error for ProbeCallError {} + +impl ProbeCallError { + /// The same error over any source error type, for a call that reads no + /// source. + fn widen(self) -> ProbeCallError { + match self { + Self::Source(never) => match never {}, + Self::Refused(refusal) => ProbeCallError::Refused(refusal), + Self::Failed(failure) => ProbeCallError::Failed(failure), + } + } +} + +/// How the host stopped a call, or how the release broke the ABI during it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ProbeFailure { + /// The instance could not be created or lacked an export. + Instantiate(String), + /// The operation name or argument does not fit a 32-bit length. + ArgumentTooLarge, + /// The release trapped, or a host call failed outside `host_read`. + Trap(String), + /// The call used its fuel. + FuelExhausted, + /// The call's reads together exceeded the total read bound. + ReadBudgetExceeded { budget: u64 }, + /// `probe_call` returned a status other than 0 or 1. + UnknownStatus(i32), + /// The result is larger than the result bound. + ResultTooLarge { bytes: u32, limit: u32 }, + /// The result lies outside linear memory. + ResultOutOfBounds, + /// The result is not JSON, or a status 1 result is not an error object. + MalformedResult(String), +} + +impl ProbeFailure { + /// An interpreter error: fuel exhaustion, or any other trap. + fn stopped(error: &wasmi::Error) -> Self { + if error.as_trap_code() == Some(TrapCode::OutOfFuel) { + Self::FuelExhausted + } else { + Self::Trap(error.to_string()) + } + } +} + +impl fmt::Display for ProbeFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Instantiate(error) => { + write!( + formatter, + "the probe release could not be instantiated: {error}" + ) + } + Self::ArgumentTooLarge => { + formatter.write_str("a probe argument exceeds a 32-bit length") + } + Self::Trap(error) => write!( + formatter, + "the probe release stopped with an error: {error}" + ), + Self::FuelExhausted => formatter.write_str("the probe release ran out of fuel"), + Self::ReadBudgetExceeded { budget } => write!( + formatter, + "the probe release read more than its {budget} byte budget" + ), + Self::UnknownStatus(status) => { + write!( + formatter, + "the probe release returned unknown status {status}" + ) + } + Self::ResultTooLarge { bytes, limit } => write!( + formatter, + "the probe release returned {bytes} bytes, over the {limit} byte limit" + ), + Self::ResultOutOfBounds => { + formatter.write_str("the probe release's result lies outside its memory") + } + Self::MalformedResult(error) => { + write!( + formatter, + "the probe release returned a malformed result: {error}" + ) + } + } + } +} + +/// A verified, compiled release. +pub(crate) struct ProbeModule { + module: Module, + linker: Linker, + descriptor: ProbeDescriptor, + limits: ProbeLimits, +} + +impl fmt::Debug for ProbeModule { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProbeModule") + .field("descriptor", &self.descriptor) + .field("limits", &self.limits) + .finish_non_exhaustive() + } +} + +impl ProbeModule { + /// Verifies `artifact` as the release `descriptor` names and compiles it. + /// + /// The descriptor is checked against `authority` first, so an unsigned + /// artifact is never parsed. The release's `version` runs once, with no + /// source, to confirm its planner. + /// + /// # Errors + /// + /// The [`ReleaseError`] naming the first check that failed. + pub(crate) async fn load( + authority: &ReleaseAuthority, + descriptor: &ProbeDescriptor, + artifact: &[u8], + limits: ProbeLimits, + ) -> Result { + authority.verify(descriptor)?; + if artifact.len() > MAX_ARTIFACT_BYTES { + return Err(ReleaseError::ArtifactTooLarge { + bytes: artifact.len(), + }); + } + if sha256_hex(artifact) != descriptor.sha256 { + return Err(ReleaseError::DigestMismatch); + } + let mut config = Config::default(); + config.consume_fuel(true); + let engine = Engine::new(&config); + let module = Module::new(&engine, artifact) + .map_err(|error| ReleaseError::InvalidModule(error.to_string()))?; + if !imports_only_host_read(&module) { + return Err(ReleaseError::ImportsMismatch); + } + let mut linker = Linker::::new(&engine); + linker + .func_wrap("env", "host_read", host_read) + .map_err(|error| ReleaseError::ExportsMismatch(error.to_string()))?; + let probe = Self { + module, + linker, + descriptor: descriptor.clone(), + limits, + }; + + let mut instance = probe + .instantiate(0) + .map_err(|error| ReleaseError::ExportsMismatch(error.to_string()))?; + let reported = instance + .abi + .call(&mut instance.store, ()) + .map_err(|error| ReleaseError::ExportsMismatch(error.to_string()))?; + if u64::from(reported) != descriptor.abi { + return Err(ReleaseError::AbiReported { reported }); + } + let version = probe + .call_without_source::("version", &json!({})) + .await + .map_err(|_| ReleaseError::PlannerMismatch)?; + if version.get("planner_id").and_then(Value::as_str) != Some(descriptor.planner_id.as_str()) + { + return Err(ReleaseError::PlannerMismatch); + } + Ok(probe) + } + + /// The verified descriptor of this release. + #[must_use] + pub(crate) fn descriptor(&self) -> &ProbeDescriptor { + &self.descriptor + } + + /// Runs `op` with `arg` over `source` and returns the release's value. + /// + /// The operation name is passed through, so every operation the release + /// implements is available. Each read the release makes is awaited from + /// `source` outside the interpreter and the suspended call resumed. + /// + /// # Errors + /// + /// [`ProbeCallError::Source`] when a read fails, + /// [`ProbeCallError::Refused`] when the release answers with its error + /// object, and [`ProbeCallError::Failed`] when a bound stops the call or + /// the release breaks the ABI. + pub(crate) async fn call( + &self, + op: &str, + arg: &Value, + source: &S, + ) -> Result> { + let mut instance = self.instantiate(source.size()).map_err(|error| { + ProbeCallError::Failed(ProbeFailure::Instantiate(error.to_string())) + })?; + let (op_pointer, op_length) = instance + .put(op.as_bytes()) + .map_err(ProbeCallError::Failed)?; + let (arg_pointer, arg_length) = instance + .put(arg.to_string().as_bytes()) + .map_err(ProbeCallError::Failed)?; + + let mut total_read = 0_u64; + let mut state = instance + .call + .call_resumable( + &mut instance.store, + (op_pointer, op_length, arg_pointer, arg_length), + ) + .map_err(stopped)?; + let status = loop { + match state { + TypedResumableCall::Finished(status) => break status, + TypedResumableCall::OutOfFuel(_) => { + return Err(ProbeCallError::Failed(ProbeFailure::FuelExhausted)); + } + TypedResumableCall::HostTrap(suspended) => { + // `host_read` is the only import, so its request is the + // only host error a call can stop with. + let Some(request) = suspended + .host_error() + .downcast_ref::() + .copied() + else { + return Err(ProbeCallError::Failed(ProbeFailure::Trap( + suspended.host_error().to_string(), + ))); + }; + let budget = self.limits.max_total_read_bytes; + total_read = total_read + .checked_add(u64::from(request.length)) + .filter(|total| *total <= budget) + .ok_or(ProbeCallError::Failed(ProbeFailure::ReadBudgetExceeded { + budget, + }))?; + let status = instance.serve(request, source).await?; + state = suspended + .resume(&mut instance.store, &[Val::I32(status)]) + .map_err(stopped)?; + } + } + }; + let result = instance + .result(self.limits.max_result_bytes) + .map_err(ProbeCallError::Failed)?; + let value: Value = serde_json::from_slice(&result).map_err(|error| { + ProbeCallError::Failed(ProbeFailure::MalformedResult(error.to_string())) + })?; + match status { + 0 => Ok(value), + 1 => Err(ProbeCallError::Refused( + refusal(&value).map_err(ProbeCallError::Failed)?, + )), + other => Err(ProbeCallError::Failed(ProbeFailure::UnknownStatus(other))), + } + } + + /// Runs `op` with `arg` and no source, for operations that read no file. + /// + /// # Errors + /// + /// As [`ProbeModule::call`], without a source error. + pub(crate) async fn call_without_source( + &self, + op: &str, + arg: &Value, + ) -> Result> { + self.call(op, arg, &NoSource) + .await + .map_err(ProbeCallError::widen) + } + + /// A fresh instance whose single import reads a source of `source_size` + /// bytes. + fn instantiate(&self, source_size: u64) -> Result { + let mut store = Store::new( + self.module.engine(), + HostState { + limits: StoreLimitsBuilder::new() + .memory_size(self.limits.memory_bytes) + .instances(1) + .memories(1) + .tables(4) + .build(), + source_size, + max_read_bytes: self.limits.max_read_bytes, + }, + ); + store.limiter(|state| &mut state.limits); + store.set_fuel(self.limits.fuel_per_call)?; + let instance = self + .linker + .instantiate_and_start(&mut store, &self.module)?; + let memory = instance + .get_memory(&store, "memory") + .ok_or_else(|| wasmi::Error::new("the release exports no memory"))?; + let _free: TypedFunc<(u32, u32), ()> = instance.get_typed_func(&store, "probe_free")?; + Ok(Instance { + abi: instance.get_typed_func(&store, "probe_abi")?, + alloc: instance.get_typed_func(&store, "probe_alloc")?, + call: instance.get_typed_func(&store, "probe_call")?, + result_ptr: instance.get_typed_func(&store, "probe_result_ptr")?, + result_len: instance.get_typed_func(&store, "probe_result_len")?, + memory, + store, + }) + } +} + +/// Whether the module's only import is `env.host_read(i64, i32, i32) -> +/// i32`. +fn imports_only_host_read(module: &Module) -> bool { + let mut imports = module.imports(); + let Some(import) = imports.next() else { + return false; + }; + let host_read = import.module() == "env" + && import.name() == "host_read" + && matches!( + import.ty(), + ExternType::Func(ty) + if ty.params() == [ValType::I64, ValType::I32, ValType::I32] + && ty.results() == [ValType::I32] + ); + host_read && imports.next().is_none() +} + +/// The release's error object, from a status 1 result. +fn refusal(value: &Value) -> Result { + let error = value + .get("error") + .ok_or_else(|| ProbeFailure::MalformedResult("a status 1 result has no error".into()))?; + let text = |field: &str| { + error + .get(field) + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| ProbeFailure::MalformedResult(format!("the error has no {field}"))) + }; + Ok(ProbeRefusal { + code: text("code")?, + integrity: error + .get("integrity") + .and_then(Value::as_bool) + .unwrap_or(false), + message: text("message")?, + }) +} + +/// A call stopped by the interpreter: out of fuel, or any other trap. +#[allow( + clippy::needless_pass_by_value, + reason = "used directly as a Result error mapper" +)] +fn stopped(error: wasmi::Error) -> ProbeCallError { + ProbeCallError::Failed(ProbeFailure::stopped(&error)) +} + +/// What the store holds for the single import. +struct HostState { + limits: StoreLimits, + source_size: u64, + max_read_bytes: u32, +} + +/// An admissible read, carried out of the interpreter as a resumable host +/// trap. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ReadRequest { + offset: u64, + length: u32, + destination: u32, +} + +impl fmt::Display for ReadRequest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "read of {} bytes at offset {} into {}", + self.length, self.offset, self.destination + ) + } +} + +impl wasmi::errors::HostError for ReadRequest {} + +/// `env.host_read`. Refuses a read over the per-read bound or past the +/// source with status 5, and suspends the call for any other. +#[allow( + clippy::needless_pass_by_value, + reason = "wasmi's host import ABI owns its Caller" +)] +fn host_read( + caller: Caller<'_, HostState>, + offset: u64, + length: u32, + destination: u32, +) -> Result { + let state = caller.data(); + let in_bounds = length <= state.max_read_bytes + && offset + .checked_add(u64::from(length)) + .is_some_and(|end| end <= state.source_size); + if !in_bounds { + return Ok(READ_OUT_OF_BOUNDS); + } + Err(wasmi::Error::host(ReadRequest { + offset, + length, + destination, + })) +} + +/// One instance, for one call. +struct Instance { + store: Store, + memory: Memory, + abi: TypedFunc<(), u32>, + alloc: TypedFunc, + call: TypedFunc<(u32, u32, u32, u32), i32>, + result_ptr: TypedFunc<(), u32>, + result_len: TypedFunc<(), u32>, +} + +impl Instance { + /// Copies `bytes` into a buffer the release allocates. + fn put(&mut self, bytes: &[u8]) -> Result<(u32, u32), ProbeFailure> { + let length = u32::try_from(bytes.len()).map_err(|_| ProbeFailure::ArgumentTooLarge)?; + let pointer = self + .alloc + .call(&mut self.store, length) + .map_err(|error| ProbeFailure::stopped(&error))?; + self.memory + .write(&mut self.store, pointer as usize, bytes) + .map_err(|error| ProbeFailure::Trap(error.to_string()))?; + Ok((pointer, length)) + } + + /// Awaits one read from `source` straight into the release's buffer and + /// returns the status to resume with. + async fn serve( + &mut self, + request: ReadRequest, + source: &S, + ) -> Result> { + let start = request.destination as usize; + let memory = self.memory.data_mut(&mut self.store); + let destination = match start.checked_add(request.length as usize) { + Some(end) => memory.get_mut(start..end), + None => None, + }; + let Some(destination) = destination else { + return Ok(READ_MEMORY_OUT_OF_BOUNDS); + }; + source + .read_exact_at(request.offset, destination) + .await + .map_err(ProbeCallError::Source)?; + Ok(READ_OK) + } + + /// The finished call's result bytes, bounded. + fn result(&mut self, limit: u32) -> Result, ProbeFailure> { + let trap = |error: wasmi::Error| ProbeFailure::stopped(&error); + let pointer = self.result_ptr.call(&mut self.store, ()).map_err(trap)?; + let length = self.result_len.call(&mut self.store, ()).map_err(trap)?; + if length > limit { + return Err(ProbeFailure::ResultTooLarge { + bytes: length, + limit, + }); + } + let mut bytes = vec![0_u8; length as usize]; + self.memory + .read(&self.store, pointer as usize, &mut bytes) + .map_err(|_| ProbeFailure::ResultOutOfBounds)?; + Ok(bytes) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/provenance/encypher/src/prepared_probe/tests.rs b/crates/provenance/encypher/src/prepared_probe/tests.rs new file mode 100644 index 0000000000..a9516890d3 --- /dev/null +++ b/crates/provenance/encypher/src/prepared_probe/tests.rs @@ -0,0 +1,766 @@ +//! The probe host against a stand-in release written in WebAssembly text. +//! +//! The stand-in implements the probe ABI with no format logic: its +//! operations exercise the host contract (two sequential reads, refused +//! reads, a read budget, fuel, memory growth, an oversized result, a trap and +//! the error object), so no Encypher planner code enters this repository. + +use std::cell::{Cell, RefCell}; +use std::fmt; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use ed25519_dalek::{Signer as _, SigningKey}; +use futures::executor::block_on; +use serde_json::json; + +use super::{ + MAX_ARTIFACT_BYTES, ProbeCallError, ProbeDescriptor, ProbeFailure, ProbeLimits, ProbeModule, + ProbeSource, ReleaseAuthority, ReleaseError, +}; +use crate::media_api::sha256_hex; + +const READ_JSON: &str = r#"{"head":"____","next":"____"}"#; +const STATUS_JSON: &str = r#"{"status":0}"#; +const UNKNOWN_JSON: &str = + r#"{"error":{"code":"UNKNOWN_OP","integrity":false,"message":"no such operation"}}"#; +const READ_FAILED_JSON: &str = + r#"{"error":{"code":"READ_FAILED","integrity":false,"message":"the host refused a read"}}"#; + +const VERSION_AT: usize = 1024; +const READ_AT: usize = 2048; +const STATUS_AT: usize = 3072; +const UNKNOWN_AT: usize = 4096; +const READ_FAILED_AT: usize = 5120; +/// WebAssembly page size, and the stand-in's initial linear memory. +const PAGE_BYTES: usize = 64 * 1024; +const INITIAL_PAGES: usize = 2; + +const STAND_IN: &str = r#" +(module + (import "env" "host_read" (func $host_read (param i64 i32 i32) (result i32))) + @EXTRA_IMPORT@ + (memory (export "memory") @INITIAL_PAGES@) + (global $heap (mut i32) (i32.const 16384)) + (global $result_ptr (mut i32) (i32.const 0)) + (global $result_len (mut i32) (i32.const 0)) + (data (i32.const @VERSION_AT@) "@VERSION_JSON@") + (data (i32.const @READ_AT@) "@READ_JSON@") + (data (i32.const @STATUS_AT@) "@STATUS_JSON@") + (data (i32.const @UNKNOWN_AT@) "@UNKNOWN_JSON@") + (data (i32.const @READ_FAILED_AT@) "@READ_FAILED_JSON@") + (func (export "probe_abi") (result i32) (i32.const @ABI@)) + (func (export "probe_alloc") (param $length i32) (result i32) + (local $pointer i32) + (local.set $pointer (global.get $heap)) + (global.set $heap (i32.add (global.get $heap) (local.get $length))) + (local.get $pointer)) + (func (export "probe_free") (param i32 i32)) + (func (export "probe_result_ptr") (result i32) (global.get $result_ptr)) + (func (export "probe_result_len") (result i32) (global.get $result_len)) + (func $answer (param $pointer i32) (param $length i32) (param $code i32) (result i32) + (global.set $result_ptr (local.get $pointer)) + (global.set $result_len (local.get $length)) + (local.get $code)) + (func $report (param $value i32) (result i32) + (i32.store8 (i32.const @STATUS_DIGIT@) (i32.add (i32.const 48) (local.get $value))) + (call $answer (i32.const @STATUS_AT@) (i32.const @STATUS_LEN@) (i32.const 0))) + (func $read_failed (result i32) + (call $answer (i32.const @READ_FAILED_AT@) (i32.const @READ_FAILED_LEN@) (i32.const 1))) + (func (export "probe_call") + (param $op i32) (param $op_len i32) (param $arg i32) (param $arg_len i32) (result i32) + (local $kind i32) + (local.set $kind (i32.load8_u (local.get $op))) + ;; "version" + (if (i32.eq (local.get $kind) (i32.const 118)) + (then (return (call $answer (i32.const @VERSION_AT@) (i32.const @VERSION_LEN@) (i32.const 0))))) + ;; "read": bytes 0..4, then bytes 4..8, into the template + (if (i32.eq (local.get $kind) (i32.const 114)) + (then + (if (call $host_read (i64.const 0) (i32.const 4) (i32.const @HEAD_SLOT@)) + (then (return (call $read_failed)))) + (if (call $host_read (i64.const 4) (i32.const 4) (i32.const @NEXT_SLOT@)) + (then (return (call $read_failed)))) + (return (call $answer (i32.const @READ_AT@) (i32.const @READ_LEN@) (i32.const 0))))) + ;; "outside": a read past the end of any test source + (if (i32.eq (local.get $kind) (i32.const 111)) + (then (return (call $report + (call $host_read (i64.const 1000000) (i32.const 4) (i32.const @HEAD_SLOT@)))))) + ;; "memory": a destination past linear memory + (if (i32.eq (local.get $kind) (i32.const 109)) + (then (return (call $report + (call $host_read (i64.const 0) (i32.const 4) (i32.const -16)))))) + ;; "greedy": reads until one is refused + (if (i32.eq (local.get $kind) (i32.const 103)) + (then + (loop $again + (br_if $again (i32.eqz + (call $host_read (i64.const 0) (i32.const 4) (i32.const @HEAD_SLOT@))))) + (return (call $read_failed)))) + ;; "Grow": memory to one page past the bound; reports 0 when refused + (if (i32.eq (local.get $kind) (i32.const 71)) + (then (return (call $report + (i32.add (i32.const 1) (memory.grow (i32.const @OVER_PAGES@))))))) + ;; "allocate": memory up to exactly the bound; reports 0 when refused + (if (i32.eq (local.get $kind) (i32.const 97)) + (then (return (call $report + (i32.add (i32.const 1) (memory.grow (i32.const @FIT_PAGES@))))))) + ;; "Beyond": one byte more than the per-read bound + (if (i32.eq (local.get $kind) (i32.const 66)) + (then (return (call $report + (call $host_read (i64.const 0) (i32.const @READ_BOUND_PLUS_ONE@) (i32.const 0)))))) + ;; "bound": exactly the per-read bound, into more than linear memory + (if (i32.eq (local.get $kind) (i32.const 98)) + (then (return (call $report + (call $host_read (i64.const 0) (i32.const @READ_BOUND@) (i32.const 0)))))) + ;; "flood": per-read-bound reads forever, ignoring every status + (if (i32.eq (local.get $kind) (i32.const 102)) + (then + (loop $again + (drop (call $host_read (i64.const 0) (i32.const @READ_BOUND@) (i32.const 0))) + (br $again)))) + ;; "echo": the argument as received + (if (i32.eq (local.get $kind) (i32.const 101)) + (then (return (call $answer (local.get $arg) (local.get $arg_len) (i32.const 0))))) + ;; "spin": never returns + (if (i32.eq (local.get $kind) (i32.const 115)) + (then (loop $forever (br $forever)))) + ;; "huge": claims a 2 GiB result + (if (i32.eq (local.get $kind) (i32.const 104)) + (then (return (call $answer (i32.const 0) (i32.const 0x7fffffff) (i32.const 0))))) + ;; "x": traps + (if (i32.eq (local.get $kind) (i32.const 120)) + (then (unreachable))) + (call $answer (i32.const @UNKNOWN_AT@) (i32.const @UNKNOWN_LEN@) (i32.const 1))) +) +"#; + +fn planner() -> String { + "a".repeat(64) +} + +fn wat_string(text: &str) -> String { + text.replace('\\', "\\\\").replace('"', "\\\"") +} + +/// The stand-in release, reporting `abi` and `planner`, optionally with a +/// second import no release may have. +fn stand_in(abi: u32, planner: &str, extra_import: bool) -> Vec { + let version = format!(r#"{{"planner_id":"{planner}"}}"#); + let head_slot = READ_AT + READ_JSON.find("____").expect("should hold a head slot"); + let next_slot = READ_AT + READ_JSON.rfind("____").expect("should hold a next slot"); + let status_digit = STATUS_AT + STATUS_JSON.find('0').expect("should hold a digit"); + let extra = if extra_import { + r#"(import "wasi_snapshot_preview1" "fd_write" (func (param i32 i32 i32 i32) (result i32)))"# + } else { + "" + }; + let edge = ProbeLimits::default(); + let fit_pages = edge.memory_bytes / PAGE_BYTES - INITIAL_PAGES; + let read_bound = edge.max_read_bytes; + let text = STAND_IN + .replace("@EXTRA_IMPORT@", extra) + .replace("@ABI@", &abi.to_string()) + .replace("@VERSION_AT@", &VERSION_AT.to_string()) + .replace("@VERSION_JSON@", &wat_string(&version)) + .replace("@VERSION_LEN@", &version.len().to_string()) + .replace("@READ_AT@", &READ_AT.to_string()) + .replace("@READ_JSON@", &wat_string(READ_JSON)) + .replace("@READ_LEN@", &READ_JSON.len().to_string()) + .replace("@HEAD_SLOT@", &head_slot.to_string()) + .replace("@NEXT_SLOT@", &next_slot.to_string()) + .replace("@STATUS_AT@", &STATUS_AT.to_string()) + .replace("@STATUS_JSON@", &wat_string(STATUS_JSON)) + .replace("@STATUS_LEN@", &STATUS_JSON.len().to_string()) + .replace("@STATUS_DIGIT@", &status_digit.to_string()) + .replace("@UNKNOWN_AT@", &UNKNOWN_AT.to_string()) + .replace("@UNKNOWN_JSON@", &wat_string(UNKNOWN_JSON)) + .replace("@UNKNOWN_LEN@", &UNKNOWN_JSON.len().to_string()) + .replace("@READ_FAILED_AT@", &READ_FAILED_AT.to_string()) + .replace("@READ_FAILED_JSON@", &wat_string(READ_FAILED_JSON)) + .replace("@READ_FAILED_LEN@", &READ_FAILED_JSON.len().to_string()) + .replace("@INITIAL_PAGES@", &INITIAL_PAGES.to_string()) + .replace("@FIT_PAGES@", &fit_pages.to_string()) + .replace("@OVER_PAGES@", &(fit_pages + 1).to_string()) + .replace("@READ_BOUND@", &read_bound.to_string()) + .replace("@READ_BOUND_PLUS_ONE@", &(read_bound + 1).to_string()); + wat::parse_str(&text).expect("should assemble the stand-in release") +} + +fn release_key() -> SigningKey { + SigningKey::from_bytes(&[7; 32]) +} + +fn signed( + key: &SigningKey, + abi: u64, + release: u64, + planner: &str, + sha256: &str, +) -> ProbeDescriptor { + let mut descriptor = ProbeDescriptor { + abi, + planner_id: planner.to_owned(), + release, + sha256: sha256.to_owned(), + signature: String::new(), + }; + let signature = key.sign(descriptor.canonical_json().as_bytes()); + descriptor.signature = STANDARD.encode(signature.to_bytes()); + descriptor +} + +fn authority(floor: u64, revoked: Vec) -> ReleaseAuthority { + ReleaseAuthority::pinned(vec![release_key().verifying_key()], floor, revoked) +} + +/// The stand-in, signed as release 3, loaded under `limits`. +fn loaded(limits: ProbeLimits) -> ProbeModule { + let artifact = stand_in(1, &planner(), false); + let descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&artifact)); + block_on(ProbeModule::load( + &authority(1, Vec::new()), + &descriptor, + &artifact, + limits, + )) + .expect("should load the signed stand-in") +} + +fn load_error(artifact: &[u8], descriptor: &ProbeDescriptor) -> ReleaseError { + block_on(ProbeModule::load( + &authority(1, Vec::new()), + descriptor, + artifact, + ProbeLimits::default(), + )) + .expect_err("should refuse the release") +} + +/// The pinned version changed under the call. +#[derive(Debug, PartialEq, Eq)] +struct VersionChanged; + +impl fmt::Display for VersionChanged { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("the source's ETag changed") + } +} + +/// Returns `Pending` once before completing, as a network read does. +#[derive(Default)] +struct YieldOnce { + yielded: bool, +} + +impl Future for YieldOnce { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<()> { + if self.yielded { + Poll::Ready(()) + } else { + self.yielded = true; + context.waker().wake_by_ref(); + Poll::Pending + } + } +} + +/// An in-memory source whose every read first suspends. +struct AsyncSource { + bytes: Vec, + size: u64, + fail: bool, + reads: RefCell>, + suspensions: Cell, +} + +impl AsyncSource { + fn new(bytes: &[u8]) -> Self { + Self { + bytes: bytes.to_vec(), + size: bytes.len() as u64, + fail: false, + reads: RefCell::new(Vec::new()), + suspensions: Cell::new(0), + } + } + + fn failing(bytes: &[u8]) -> Self { + Self { + fail: true, + ..Self::new(bytes) + } + } + + /// A source that claims `size` bytes and holds none, for reads the host + /// must refuse before asking it. + fn claiming(size: u64) -> Self { + Self { + size, + ..Self::new(&[]) + } + } + + fn reads(&self) -> Vec<(u64, usize)> { + self.reads.borrow().clone() + } +} + +impl ProbeSource for AsyncSource { + type Error = VersionChanged; + + fn size(&self) -> u64 { + self.size + } + + async fn read_exact_at( + &self, + offset: u64, + destination: &mut [u8], + ) -> Result<(), VersionChanged> { + self.suspensions.set(self.suspensions.get() + 1); + YieldOnce::default().await; + self.reads.borrow_mut().push((offset, destination.len())); + if self.fail { + return Err(VersionChanged); + } + let start = usize::try_from(offset).expect("should fit a test offset"); + destination.copy_from_slice(&self.bytes[start..start + destination.len()]); + Ok(()) + } +} + +/// Release 8 as the production signing service advertised it, signed by +/// Encypher's production release key. Only the public descriptor is kept; +/// its artifact is not part of this repository. +const PRODUCTION_RELEASE_8: &str = r#"{ + "abi": 1, + "planner_id": "0685f74f2b47b171a62be7d8f38224c85b5fdc1fc849d316e8c93910406ece04", + "release": 8, + "sha256": "45c8e965b33e6763a52a9164392fb584f1dd6d993e9583a9f51945caa4ebed59", + "signature": "Mge6x1crHLQtuvn0eDjtapacOVTiEfU7/GOoxRd09e2bFGZaiSn9CvBlf/hSZs6Auvb2aRhWDVeOqnw+iTMMCw==" +}"#; + +#[test] +fn production_authority_accepts_only_releases_encypher_signed() { + let authority = ReleaseAuthority::production(); + let published: serde_json::Value = + serde_json::from_str(PRODUCTION_RELEASE_8).expect("should parse the fixture"); + let published = ProbeDescriptor::parse(&published).expect("should be a descriptor"); + assert_eq!( + authority.select(std::slice::from_ref(&published)), + Ok(&published), + "should accept the release Encypher published" + ); + + let mut retargeted = published.clone(); + retargeted.sha256 = "b".repeat(64); + assert_eq!( + authority.verify(&retargeted), + Err(ReleaseError::Unsigned { release: 8 }), + "should refuse the published signature over another artifact" + ); + let foreign = signed( + &release_key(), + 1, + 8, + &published.planner_id, + &published.sha256, + ); + assert_eq!( + authority.verify(&foreign), + Err(ReleaseError::Unsigned { release: 8 }), + "should refuse the same release signed by any other key" + ); +} + +#[test] +fn descriptors_parse_only_in_the_contract_shape() { + let descriptor = signed(&release_key(), 1, 3, &planner(), &"b".repeat(64)); + assert_eq!( + ProbeDescriptor::parse(&descriptor.to_json()), + Some(descriptor.clone()) + ); + assert_eq!( + descriptor.canonical_json(), + format!( + r#"{{"abi":1,"planner_id":"{}","release":3,"sha256":"{}"}}"#, + planner(), + "b".repeat(64) + ), + "should sign the four fields, sorted, without whitespace" + ); + assert_eq!( + descriptor.artifact_path(), + format!("sign/media/probe/{}.wasm", "b".repeat(64)) + ); + + let mut upper = descriptor.to_json(); + upper["sha256"] = json!("B".repeat(64)); + let mut short = descriptor.to_json(); + short["planner_id"] = json!("a".repeat(63)); + let mut unsigned = descriptor.to_json(); + unsigned["signature"] = json!(""); + let mut textual = descriptor.to_json(); + textual["release"] = json!("3"); + for malformed in [&upper, &short, &unsigned, &textual] { + assert_eq!( + ProbeDescriptor::parse(malformed), + None, + "should refuse {malformed}" + ); + } + let list = json!([descriptor.to_json(), upper, "noise"]); + assert_eq!(ProbeDescriptor::parse_list(Some(&list)), vec![descriptor]); + assert!(ProbeDescriptor::parse_list(Some(&json!({}))).is_empty()); +} + +#[test] +fn verification_refuses_every_tampered_descriptor() { + let authority = authority(2, vec!["c".repeat(64)]); + let good = signed(&release_key(), 1, 3, &planner(), &"b".repeat(64)); + assert_eq!(authority.verify(&good), Ok(())); + + let mut retargeted = good.clone(); + retargeted.sha256 = "d".repeat(64); + assert_eq!( + authority.verify(&retargeted), + Err(ReleaseError::Unsigned { release: 3 }) + ); + + let foreign = signed( + &SigningKey::from_bytes(&[9; 32]), + 1, + 3, + &planner(), + &"b".repeat(64), + ); + assert_eq!( + authority.verify(&foreign), + Err(ReleaseError::Unsigned { release: 3 }) + ); + + for signature in ["not base64!", "AAAA", &STANDARD.encode([0_u8; 63])] { + let mut garbled = good.clone(); + garbled.signature = signature.to_owned(); + assert_eq!( + authority.verify(&garbled), + Err(ReleaseError::MalformedSignature { release: 3 }), + "should refuse signature {signature:?}" + ); + } + + let future_abi = signed(&release_key(), 2, 3, &planner(), &"b".repeat(64)); + assert_eq!( + authority.verify(&future_abi), + Err(ReleaseError::AbiUnsupported { abi: 2 }) + ); + assert_eq!( + authority.verify(&future_abi).map_err(|error| error.code()), + Err("PROBE_UPGRADE_REQUIRED") + ); + + let old = signed(&release_key(), 1, 1, &planner(), &"b".repeat(64)); + assert_eq!( + authority.verify(&old), + Err(ReleaseError::BelowFloor { + release: 1, + floor: 2 + }) + ); + + let withdrawn = signed(&release_key(), 1, 4, &planner(), &"C".repeat(64)); + assert_eq!( + authority.verify(&withdrawn), + Err(ReleaseError::Revoked { release: 4 }) + ); +} + +#[test] +fn selection_takes_the_newest_compatible_release_and_never_falls_back() { + let authority = authority(1, Vec::new()); + let older = signed(&release_key(), 1, 3, &planner(), &"b".repeat(64)); + let newer = signed(&release_key(), 1, 5, &planner(), &"c".repeat(64)); + let future = signed(&release_key(), 2, 9, &planner(), &"d".repeat(64)); + let advertised = [older.clone(), future.clone(), newer.clone()]; + assert_eq!(authority.select(&advertised), Ok(&newer)); + + assert_eq!(authority.select(&[]), Err(ReleaseError::NoneAdvertised)); + assert_eq!( + authority.select(std::slice::from_ref(&future)), + Err(ReleaseError::NoCompatibleRelease) + ); + + let mut tampered = newer; + tampered.planner_id = "e".repeat(64); + assert_eq!( + authority.select(&[older, tampered]), + Err(ReleaseError::Unsigned { release: 5 }), + "should refuse the newest rather than run an older release" + ); +} + +#[test] +fn loading_refuses_artifacts_that_are_not_the_signed_release() { + let artifact = stand_in(1, &planner(), false); + let descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&artifact)); + + let mut tampered = artifact.clone(); + let last = tampered.len() - 1; + tampered[last] ^= 1; + assert_eq!( + load_error(&tampered, &descriptor), + ReleaseError::DigestMismatch + ); + + let oversized = vec![0_u8; MAX_ARTIFACT_BYTES + 1]; + let oversized_descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&oversized)); + assert_eq!( + load_error(&oversized, &oversized_descriptor), + ReleaseError::ArtifactTooLarge { + bytes: MAX_ARTIFACT_BYTES + 1 + } + ); + + let garbage = b"not a module".to_vec(); + let garbage_descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&garbage)); + assert!(matches!( + load_error(&garbage, &garbage_descriptor), + ReleaseError::InvalidModule(_) + )); + let mut unsigned = garbage_descriptor; + unsigned.release = 4; + assert_eq!( + load_error(&garbage, &unsigned), + ReleaseError::Unsigned { release: 4 }, + "should refuse the descriptor before parsing its artifact" + ); +} + +#[test] +fn loading_refuses_a_signed_release_that_breaks_the_abi() { + let importing = stand_in(1, &planner(), true); + let descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&importing)); + assert_eq!( + load_error(&importing, &descriptor), + ReleaseError::ImportsMismatch + ); + + let misreporting = stand_in(2, &planner(), false); + let descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&misreporting)); + assert_eq!( + load_error(&misreporting, &descriptor), + ReleaseError::AbiReported { reported: 2 } + ); + + let other_planner = stand_in(1, &"f".repeat(64), false); + let descriptor = signed( + &release_key(), + 1, + 3, + &planner(), + &sha256_hex(&other_planner), + ); + assert_eq!( + load_error(&other_planner, &descriptor), + ReleaseError::PlannerMismatch + ); +} + +#[test] +fn reads_are_awaited_in_order_and_the_call_resumes_without_replay() { + let probe = loaded(ProbeLimits::default()); + assert_eq!(probe.descriptor().release, 3); + let source = AsyncSource::new(b"abcdefgh-rest-of-a-large-file"); + + let value = block_on(probe.call("read", &json!({}), &source)).expect("should read"); + + assert_eq!(value, json!({"head": "abcd", "next": "efgh"})); + assert_eq!( + source.reads(), + vec![(0, 4), (4, 4)], + "each read should reach the source once, in order" + ); + assert_eq!( + source.suspensions.get(), + 2, + "each read should suspend the call" + ); + + let again = block_on(probe.call("read", &json!({}), &source)).expect("should read again"); + assert_eq!( + again, value, + "a second call should start from a fresh instance" + ); + assert_eq!(source.reads().len(), 4); +} + +#[test] +fn the_argument_reaches_the_release_intact() { + let probe = loaded(ProbeLimits::default()); + let arg = json!({"plan": {"items": [1, 2, 3]}, "text": "caf\u{e9} \"quoted\""}); + + let echoed = + block_on(probe.call_without_source::("echo", &arg)).expect("should echo"); + + assert_eq!(echoed, arg); +} + +#[test] +fn reads_outside_the_source_or_memory_are_refused_without_asking_the_source() { + let probe = loaded(ProbeLimits::default()); + let source = AsyncSource::new(b"abcdefgh"); + + let outside = block_on(probe.call("outside", &json!({}), &source)).expect("should answer"); + assert_eq!(outside, json!({"status": 5})); + + let memory = block_on(probe.call("memory", &json!({}), &source)).expect("should answer"); + assert_eq!(memory, json!({"status": 4})); + + assert!( + source.reads().is_empty(), + "no refused read should reach the source" + ); +} + +#[test] +fn the_edge_per_read_bound_admits_its_size_and_refuses_one_byte_more() { + let probe = loaded(ProbeLimits::default()); + let bound = u64::from(ProbeLimits::default().max_read_bytes); + let source = AsyncSource::claiming(bound + 1); + + let beyond = block_on(probe.call("Beyond", &json!({}), &source)).expect("should answer"); + assert_eq!( + beyond, + json!({"status": 5}), + "a read over the bound should be refused" + ); + + let at_bound = block_on(probe.call("bound", &json!({}), &source)).expect("should answer"); + assert_eq!( + at_bound, + json!({"status": 4}), + "a read at the bound should pass the bound and stop only at linear memory" + ); + assert!( + source.reads().is_empty(), + "neither read should reach the source" + ); +} + +#[test] +fn the_edge_total_read_budget_stops_a_release_that_keeps_reading() { + let probe = loaded(ProbeLimits::default()); + let source = AsyncSource::claiming(u64::from(ProbeLimits::default().max_read_bytes)); + + let error = block_on(probe.call("flood", &json!({}), &source)).expect_err("should stop"); + + assert!( + matches!( + error, + ProbeCallError::Failed(ProbeFailure::ReadBudgetExceeded { .. }) + ), + "the read budget, not fuel, should stop the release: {error}" + ); +} + +#[test] +fn the_read_budget_counts_every_admitted_read() { + let probe = loaded(ProbeLimits { + max_total_read_bytes: 8, + ..ProbeLimits::default() + }); + let source = AsyncSource::new(b"abcdefgh"); + + let error = block_on(probe.call("greedy", &json!({}), &source)).expect_err("should stop"); + + assert!(matches!( + error, + ProbeCallError::Failed(ProbeFailure::ReadBudgetExceeded { budget: 8 }) + )); + assert_eq!(source.reads(), vec![(0, 4), (0, 4)]); +} + +#[test] +fn a_failed_read_abandons_the_call_with_the_source_error() { + let probe = loaded(ProbeLimits::default()); + let source = AsyncSource::failing(b"abcdefgh"); + + let error = block_on(probe.call("read", &json!({}), &source)).expect_err("should fail"); + + assert!(matches!(error, ProbeCallError::Source(VersionChanged))); + assert_eq!( + source.reads(), + vec![(0, 4)], + "no read should follow the failure" + ); +} + +#[test] +fn edge_fuel_stops_a_release_that_never_returns() { + let probe = loaded(ProbeLimits::default()); + + let spin = block_on(probe.call_without_source::("spin", &json!({}))); + + assert!(matches!( + spin, + Err(ProbeCallError::Failed(ProbeFailure::FuelExhausted)) + )); +} + +#[test] +fn edge_memory_grows_to_its_bound_and_no_further() { + let probe = loaded(ProbeLimits::default()); + + let to_bound = block_on(probe.call_without_source::("allocate", &json!({}))) + .expect("should answer"); + assert_eq!( + to_bound, + json!({"status": 3}), + "growth to exactly the bound should succeed" + ); + + let past_bound = block_on(probe.call_without_source::("Grow", &json!({}))) + .expect("should answer"); + assert_eq!( + past_bound, + json!({"status": 0}), + "growth one page past the bound should fail inside the release" + ); +} + +#[test] +fn an_oversized_result_is_refused() { + let probe = loaded(ProbeLimits::default()); + + let huge = block_on(probe.call_without_source::("huge", &json!({}))); + + assert!(matches!( + huge, + Err(ProbeCallError::Failed(ProbeFailure::ResultTooLarge { .. })) + )); +} + +#[test] +fn traps_and_error_objects_are_reported_as_such() { + let probe = loaded(ProbeLimits::default()); + + let trap = block_on(probe.call_without_source::("x", &json!({}))); + assert!(matches!( + trap, + Err(ProbeCallError::Failed(ProbeFailure::Trap(_))) + )); + + let unknown = block_on(probe.call_without_source::("unknown", &json!({}))) + .expect_err("should refuse"); + assert_eq!(unknown.code(), "UNKNOWN_OP"); + let ProbeCallError::Refused(refusal) = unknown else { + panic!("should carry the release's error object"); + }; + assert!(!refusal.integrity); + assert_eq!(refusal.message, "no such operation"); +} diff --git a/crates/provenance/encypher/src/processor.rs b/crates/provenance/encypher/src/processor.rs new file mode 100644 index 0000000000..3604e0388f --- /dev/null +++ b/crates/provenance/encypher/src/processor.rs @@ -0,0 +1,3305 @@ +//! Per-document signing flow: gates, hold, KV cache, sign calls, outcomes, logging. +//! +//! A document moves through four phases: +//! +//! 1. **Admit** (the middleware's preparation, with no I/O): the request +//! gates decide whether the document is held at all. +//! 2. **Extract** (synchronous): at end of input the held document's article +//! boundary, visible text and text hash are computed, and its embedded +//! media found. +//! 3. **Resolve** (the asynchronous finish): policy and plan caches, the policy +//! and sign calls, and cache writes, then each embedded media item signed +//! through [`media`]. On runtimes with a timer every call is raced against +//! one `max_hold_ms` budget, keeping a reserve for the skip write. +//! 4. **Render** (synchronous): the plan's markers and the manifest meta tag are +//! applied to the held document, then each signed media item replaces its +//! exact bytes. +//! +//! Every module failure emits the held document unchanged, or, for one +//! media item, leaves that item as written and reports it unsigned. + +use std::borrow::Cow; +use std::io; +use std::pin::pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use futures::future::{Either, LocalBoxFuture, select}; +use http::{HeaderValue, Method, Request, StatusCode, header}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +use crate::api::{self, ApiError, EmbeddingPlan, SignRequest}; +use crate::boundary::{self, Boundary}; +use crate::embedded; +use crate::{EncypherConfig, INTEGRATION_ID, embed, text}; +use trusted_server_core::integrations::ensure_integration_backend_with_timeout; +use trusted_server_core::platform::{ + KvError, PlatformHttpRequest, PlatformKvStore, PlatformTimer, RuntimeServices, +}; +use trusted_server_core::streaming_processor::{AsyncFinish, StreamProcessor}; + +mod media; +mod prepared; + +pub(crate) use media::EncypherMedia; + +const SIGN_PATH: &str = "/api/v1/public/cdn/sign"; +const POLICY_PATH: &str = "/api/v1/public/cdn/trusted-server/policy"; +const WORKER_TOKEN_HEADER: &str = "x-encypher-worker-token"; +/// Successful article plans. Entries carry their own `expires_at`. +const PLAN_KEY_PREFIX: &str = "encypher:plan:v4"; +/// Article failures, separate so a delayed failure cannot replace a plan. +const PLAN_SKIP_KEY_PREFIX: &str = "encypher:plan-skip:v4"; +/// Successful policy digests. Entries carry their own `expires_at`. +const POLICY_KEY_PREFIX: &str = "encypher:policy:v3"; +/// Policy failures, separate so a delayed failure cannot replace a digest. +const POLICY_SKIP_KEY_PREFIX: &str = "encypher:policy-skip:v3"; +/// Part of `max_hold_ms` kept back from resolve work so a skip can still be +/// written after the work runs out of time. +const SKIP_WRITE_RESERVE: Duration = Duration::from_millis(200); +/// How far past `max_hold_ms` the core backstop deadline lies. The module +/// bounds itself first; the backstop only catches an overrun. +const BACKSTOP_MARGIN: Duration = Duration::from_millis(1_000); + +/// Path prefixes never signed (ported from the Fastly worker's `shouldSkipPath`). +const SKIP_PATH_PREFIXES: &[&str] = &[ + "/api/", + "/wp-admin/", + "/wp-json/", + "/feed/", + "/sitemap", + "/.well-known/", +]; +/// Asset suffixes the article path never signs (ported from the Fastly +/// worker's `shouldSkipPath`). Media at these paths is signed by [`media`]. +const SKIP_PATH_SUFFIXES: &[&str] = &[ + ".xml", ".json", ".rss", ".atom", ".css", ".js", ".woff", ".woff2", ".png", ".jpg", ".gif", + ".svg", ".ico", +]; + +/// Set once this instance has warned that the runtime KV store is unbound. +static KV_UNAVAILABLE_WARNED: AtomicBool = AtomicBool::new(false); + +/// Why a document was not held. Evaluated in the middleware's preparation, +/// with no I/O. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Gate { + Method, + Status, + OriginPrivate, + InvalidPageUrl, + HostMismatch, + ExcludedPath, +} + +impl Gate { + pub(crate) fn code(self) -> &'static str { + match self { + Self::Method => "method", + Self::Status => "status", + Self::OriginPrivate => "origin_private", + Self::InvalidPageUrl => "invalid_page_url", + Self::HostMismatch => "host_mismatch", + Self::ExcludedPath => "excluded_path", + } + } +} + +/// What the gates are told about the page a document is the body of. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PageRequest { + /// The reader's request method. + pub(crate) method: Method, + /// The scheme, host and path the reader asked for, without the query, + /// which is the address the article is signed for. On a host that + /// serves several publishers by path it includes the publisher's prefix. + pub(crate) page_url: String, + /// The path prefix request addressing removed, such as + /// `/www.example.com`, or empty. The path gates read the publisher's own + /// path, without it. + pub(crate) removed_prefix: String, + /// The status of the response the document is the body of. + pub(crate) response_status: u16, + /// Whether the origin marked its response as one reader's. + pub(crate) origin_response_is_private: bool, +} + +/// Reason code for a held document's or a media item's outcome. Sign +/// failures are also stored as the reason of a negative (`skip`) cache entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Reason { + Signed, + CacheHit, + DocumentTooLarge, + OutputTooLarge, + NotUtf8, + NoBoundary, + AlreadySigned, + ManifestPresent, + TextTooShort, + KvUnavailable, + KvError, + Timeout, + CachedSkip, + CachedEntryInvalid, + CachedPlanInvalid, + RequestBuild, + Backend, + HttpError, + Forbidden, + Unprocessable, + RateLimited, + ServerError, + UnexpectedStatus, + ResponseTooLarge, + MalformedResponse, + PolicyChanged, + PolicyDigestMismatch, + QuotaExceeded, + SignRejected, + NoPlan, + InvalidPlan, + AlignmentFailed, + /// The media is over `max_media_bytes`. + MediaTooLarge, + /// The media is delivered outside the claimed domain. + OutsideClaim, + /// Core named no single hostname the source was fetched from. + InvalidOriginHost, + /// The captured bytes are not the bytes core hashed. + SourceDigestMismatch, + /// A `data:` URI's payload does not decode. + MalformedDataUri, + /// The endpoint answered `409` to a reused replay key. + IdempotencyConflict, + /// The endpoint no longer holds the result for the replay key. + ResultPurged, + /// The endpoint refused the request body's size. + PayloadTooLarge, + /// The signed bytes are of another media type than the source. + SignedTypeMismatch, + /// The signed bytes are not the bytes the answer names. + SignedDigestMismatch, + /// A signing header is missing, repeated or malformed. + InvalidSignedResponse, + /// The manifest is remote and needs a `Link` the delivery cannot carry. + ManifestUndeliverable, + /// The answer carries no manifest and no reference to one. + UnsignedResponse, + /// The response's policies are not visible to the module, so an inline + /// style edit could break a style hash. + CspUnknown, + /// A page policy admits the inline style by hash. + CspStyleHash, + /// The element carries a subresource integrity value. + IntegrityAttribute, + /// The URI is written with an HTML character reference. + CharacterReference, + /// The URI is written with a CSS escape. + CssEscape, + /// No standalone SVG document is proven to parse to the tree the browser + /// built from the inline SVG. + InlineSvgNotEquivalent, + /// The inline SVG carries provenance of its own and is not, as written, + /// a standalone SVG equivalent to the browser's tree, so it is never + /// rewritten. + SvgProvenanceNotEquivalent, + /// Original SVG provenance protects the entire range from inline edits + /// or replacement with a new, unchained signature. + SvgExistingProvenance, + /// The signer changed the inline SVG beyond adding its manifest. + InlineSvgAltered, + /// The document's media made its most endpoint calls; the item is signed + /// on a later view. + Deferred, +} + +impl Reason { + pub(crate) fn code(self) -> &'static str { + match self { + Self::Signed => "signed", + Self::CacheHit => "cache_hit", + Self::DocumentTooLarge => "document_too_large", + Self::OutputTooLarge => "output_too_large", + Self::NotUtf8 => "not_utf8", + Self::NoBoundary => "no_boundary", + Self::AlreadySigned => "already_signed", + Self::ManifestPresent => "manifest_present", + Self::TextTooShort => "text_too_short", + Self::KvUnavailable => "kv_unavailable", + Self::KvError => "kv_error", + Self::Timeout => "timeout", + Self::CachedSkip => "cached_skip", + Self::CachedEntryInvalid => "cached_entry_invalid", + Self::CachedPlanInvalid => "cached_plan_invalid", + Self::RequestBuild => "request_build", + Self::Backend => "backend", + Self::HttpError => "http_error", + Self::Forbidden => "forbidden", + Self::Unprocessable => "unprocessable", + Self::RateLimited => "rate_limited", + Self::ServerError => "server_error", + Self::UnexpectedStatus => "unexpected_status", + Self::ResponseTooLarge => "response_too_large", + Self::MalformedResponse => "malformed_response", + Self::PolicyChanged => "policy_changed", + Self::PolicyDigestMismatch => "policy_digest_mismatch", + Self::QuotaExceeded => "quota_exceeded", + Self::SignRejected => "sign_rejected", + Self::NoPlan => "no_plan", + Self::InvalidPlan => "invalid_plan", + Self::AlignmentFailed => "alignment_failed", + Self::MediaTooLarge => "media_too_large", + Self::OutsideClaim => "outside_claim", + Self::InvalidOriginHost => "invalid_origin_host", + Self::SourceDigestMismatch => "source_digest_mismatch", + Self::MalformedDataUri => "malformed_data_uri", + Self::IdempotencyConflict => "idempotency_conflict", + Self::ResultPurged => "result_purged", + Self::PayloadTooLarge => "payload_too_large", + Self::SignedTypeMismatch => "signed_type_mismatch", + Self::SignedDigestMismatch => "signed_digest_mismatch", + Self::InvalidSignedResponse => "invalid_signed_response", + Self::ManifestUndeliverable => "manifest_undeliverable", + Self::UnsignedResponse => "unsigned_response", + Self::CspUnknown => "csp_unknown", + Self::CspStyleHash => "csp_style_hash", + Self::IntegrityAttribute => "integrity_attribute", + Self::CharacterReference => "character_reference", + Self::CssEscape => "css_escape", + Self::InlineSvgNotEquivalent => "inline_svg_not_equivalent", + Self::SvgProvenanceNotEquivalent => "svg_provenance_not_equivalent", + Self::SvgExistingProvenance => "svg_existing_provenance", + Self::InlineSvgAltered => "inline_svg_altered", + Self::Deferred => "deferred", + } + } +} + +/// A cache entry that records when it stops being usable. +trait Expiring { + /// Unix seconds at which the entry expires. + fn expires_at(&self) -> u64; + /// Whether the entry was written without a store TTL, so it stays until deleted. + fn ttl_less(&self) -> bool; + /// Record that the entry is written without a store TTL. + fn mark_ttl_less(&mut self); +} + +/// An article cache entry, keyed by credential context, policy and text hash. +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum CacheEntry { + Plan { + plan: EmbeddingPlan, + manifest_url: Option, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Skip { + reason: Reason, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, +} + +impl Expiring for CacheEntry { + fn expires_at(&self) -> u64 { + match self { + Self::Plan { expires_at, .. } | Self::Skip { expires_at, .. } => *expires_at, + } + } + + fn ttl_less(&self) -> bool { + match self { + Self::Plan { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less, + } + } + + fn mark_ttl_less(&mut self) { + match self { + Self::Plan { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less = true, + } + } +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum PolicyEntry { + Digest { + policy_digest: String, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Skip { + reason: Reason, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, +} + +impl Expiring for PolicyEntry { + fn expires_at(&self) -> u64 { + match self { + Self::Digest { expires_at, .. } | Self::Skip { expires_at, .. } => *expires_at, + } + } + + fn ttl_less(&self) -> bool { + match self { + Self::Digest { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less, + } + } + + fn mark_ttl_less(&mut self) { + match self { + Self::Digest { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less = true, + } + } +} + +/// The one freshness predicate applied to every cache read. +fn is_fresh(expires_at: u64, now: u64) -> bool { + now < expires_at +} + +/// Current Unix time in seconds, from a clock that works on every adapter. +fn unix_now() -> u64 { + web_time::SystemTime::now() + .duration_since(web_time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) +} + +/// Configuration shared by every document processor. +pub(crate) struct Signer { + config: EncypherConfig, + max_document_bytes: usize, + sign_url: String, + media_sign_url: String, + policy_url: String, + policy_key: String, + policy_skip_key: String, + /// First 16 hex characters of `sha256(api_base_url \n domain \n token)`. + cache_context: String, + /// Set by the first TTL write the store reports as unsupported, or by reading + /// an entry written without a TTL; from then on this instance writes without + /// a TTL and deletes expired entries it reads. + ttl_less: AtomicBool, +} + +impl Signer { + /// The signer for `config`, holding at most `max_document_bytes` of one + /// document. + pub(crate) fn new(config: EncypherConfig, max_document_bytes: usize) -> Self { + let sign_url = format!("{}{SIGN_PATH}", config.api_base_url.trim_end_matches('/')); + let media_sign_url = format!( + "{}{}", + config.api_base_url.trim_end_matches('/'), + crate::media_api::MEDIA_SIGN_PATH + ); + let policy_url = format!("{}{POLICY_PATH}", config.api_base_url.trim_end_matches('/')); + let context_hash = text::content_hash_hex(&format!( + "{}\n{}\n{}", + config.api_base_url, + config.domain, + config.worker_token.expose().trim() + )); + let cache_context = context_hash + .get(..16) + .unwrap_or(context_hash.as_str()) + .to_owned(); + Self { + config, + max_document_bytes, + sign_url, + media_sign_url, + policy_url, + policy_key: format!("{POLICY_KEY_PREFIX}:{cache_context}"), + policy_skip_key: format!("{POLICY_SKIP_KEY_PREFIX}:{cache_context}"), + cache_context, + ttl_less: AtomicBool::new(false), + } + } +} + +/// What a held document is, which decides what in it is signed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Surface { + /// An HTML page: its article's text, then its embedded media. + Page, + /// A stylesheet: the media its `url()`s embed. + Stylesheet, +} + +/// Holds the document of an admitted page until it ends. `policies` are the +/// response's `Content-Security-Policy` values, `None` when core cannot +/// show them, which decide whether inline style media may be edited. +pub(crate) fn document_processor( + signer: &Arc, + request: PageRequest, + policies: Option>, +) -> DocumentProcessor { + held_processor(signer, request, Surface::Page, policies) +} + +/// Holds an admitted stylesheet until it ends. +pub(crate) fn stylesheet_processor( + signer: &Arc, + request: PageRequest, +) -> DocumentProcessor { + held_processor(signer, request, Surface::Stylesheet, None) +} + +fn held_processor( + signer: &Arc, + request: PageRequest, + surface: Surface, + policies: Option>, +) -> DocumentProcessor { + DocumentProcessor { + signer: Arc::clone(signer), + request, + surface, + policies, + held: Vec::new(), + state: DocumentState::Holding, + } +} + +/// What a held document's signing works for. +struct Target<'a> { + request: &'a PageRequest, + surface: Surface, + /// The response's policy values, `None` when core cannot show them. + policies: Option<&'a [String]>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DocumentState { + /// Accumulating the document. + Holding, + /// Over the size limit: chunks pass through and nothing is signed. + PassingThrough, + /// The whole document was emitted as held output; it may be signed in the + /// asynchronous finish. + Complete, +} + +/// Holds one admitted document until end of input, then signs it in the +/// asynchronous finish or passes it through. +pub(crate) struct DocumentProcessor { + signer: Arc, + request: PageRequest, + surface: Surface, + policies: Option>, + held: Vec, + state: DocumentState, +} + +impl StreamProcessor for DocumentProcessor { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error> { + if self.state != DocumentState::Holding { + return Ok(chunk.to_vec()); + } + if self.held.len().saturating_add(chunk.len()) > self.signer.max_document_bytes { + self.state = DocumentState::PassingThrough; + log_outcome(false, Reason::DocumentTooLarge); + let mut output = std::mem::take(&mut self.held); + output.extend_from_slice(chunk); + return Ok(output); + } + self.held.extend_from_slice(chunk); + if !is_last { + return Ok(Vec::new()); + } + // The complete document is the fail-open output; the driver owns it. + self.state = DocumentState::Complete; + Ok(std::mem::take(&mut self.held)) + } + + fn reset(&mut self) { + self.held.clear(); + self.state = DocumentState::Holding; + } + + fn async_finish<'a>( + &'a mut self, + held: &'a [u8], + services: &'a RuntimeServices, + ) -> Option> { + if self.state != DocumentState::Complete { + return None; + } + let Ok(document) = std::str::from_utf8(held) else { + log_outcome(false, Reason::NotUtf8); + return None; + }; + let (article, media) = match self.surface { + Surface::Page => ( + Some(self.signer.extract(document)), + embedded::scan_html(document, self.policies.as_deref()), + ), + Surface::Stylesheet => (None, embedded::scan_stylesheet(document)), + }; + let article = match article { + Some(Ok(article)) => Some(article), + Some(Err(reason)) => { + log_outcome(false, reason); + None + } + None => None, + }; + if article.is_none() && media.is_empty() { + return None; + } + let signer = &*self.signer; + let target = Target { + request: &self.request, + surface: self.surface, + policies: self.policies.as_deref(), + }; + let deadline = + Duration::from_millis(signer.config.max_hold_ms).saturating_add(BACKSTOP_MARGIN); + Some(AsyncFinish::new(deadline, async move { + signer + .resolve_document(&target, services, document, article, media) + .await + })) + } +} + +/// The article found in a held document. +struct Article { + boundary: Boundary, + /// Visible text sent for signing, truncated to the API limit. + text: String, + /// Unicode scalars in `text`. + scalars: usize, + /// First structural closing-head offset in the original document. + head_close: Option, + /// First completed structural title's inner byte range. + title_range: Option<(usize, usize)>, +} + +/// Time a document's resolve may still take, on runtimes with a timer. +enum Deadline<'a> { + /// No timer: calls are bounded only by backend timeouts. + Unbounded, + /// The timer has not fired yet. + Running(LocalBoxFuture<'a, ()>), + /// The timer fired; every further race loses at once. + Expired, +} + +impl<'a> Deadline<'a> { + fn new(timer: Option<&'a dyn PlatformTimer>, budget: Duration) -> Self { + timer.map_or(Self::Unbounded, |timer| Self::Running(timer.sleep(budget))) + } + + /// Start the clock now. A timer's sleep starts on its first poll, which + /// would otherwise be the first call that does not complete at once. + async fn start(&mut self) { + if let Self::Running(sleep) = self + && futures::poll!(sleep.as_mut()).is_ready() + { + *self = Self::Expired; + } + } + + /// Run `work` to completion, or return `None` once the deadline passes. + async fn race(&mut self, work: impl Future) -> Option { + match self { + Self::Unbounded => return Some(work.await), + Self::Expired => return None, + Self::Running(sleep) => { + if let Either::Left((value, _)) = select(pin!(work), sleep.as_mut()).await { + return Some(value); + } + } + } + *self = Self::Expired; + None + } + + /// Whether the deadline has passed, so the last race lost. + fn expired(&self) -> bool { + matches!(self, Self::Expired) + } +} + +fn log_outcome(signed: bool, reason: Reason) { + if reason == Reason::KvUnavailable && !KV_UNAVAILABLE_WARNED.swap(true, Ordering::Relaxed) { + log::warn!( + "encypher: the runtime KV store is unavailable, so pages pass through unsigned; bind the adapter's KV store to enable signing" + ); + } + log::info!( + "encypher: outcome={} reason={}", + if signed { "signed" } else { "passed_through" }, + reason.code() + ); +} + +/// Whether `path` is one the article path never signs: an API, feed, admin +/// or well-known path, or an asset. +pub(crate) fn is_excluded_path(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + is_excluded_prefix(&lower) + || SKIP_PATH_SUFFIXES + .iter() + .any(|suffix| lower.ends_with(suffix)) +} + +/// Whether `path` is an API, feed, admin or well-known path, where no media +/// is signed either. Asset suffixes are not excluded: media is. +pub(crate) fn is_excluded_prefix(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + SKIP_PATH_PREFIXES + .iter() + .any(|prefix| lower.starts_with(prefix)) +} + +impl Signer { + /// Apply the request gates to a page. Only an admitted document is held. + pub(crate) fn admit(&self, request: PageRequest) -> Result { + self.gate(request, true) + } + + /// Apply the request gates to a stylesheet, whose `.css` path the page + /// gates exclude. + pub(crate) fn admit_stylesheet(&self, request: PageRequest) -> Result { + self.gate(request, false) + } + + fn gate(&self, request: PageRequest, page: bool) -> Result { + if request.method != Method::GET { + return Err(Gate::Method); + } + if request.response_status != StatusCode::OK.as_u16() { + return Err(Gate::Status); + } + if request.origin_response_is_private { + return Err(Gate::OriginPrivate); + } + let page_url = url::Url::parse(&request.page_url).map_err(|_| Gate::InvalidPageUrl)?; + let host = page_url.host_str().ok_or(Gate::InvalidPageUrl)?; + if !self.claims_host(host) { + return Err(Gate::HostMismatch); + } + // The publisher's own path, as its site names it. + let path = page_url.path(); + let path = path + .strip_prefix(request.removed_prefix.as_str()) + .filter(|rest| rest.starts_with('/')) + .unwrap_or(path); + let excluded = if page { + is_excluded_path(path) + } else { + is_excluded_prefix(path) + }; + if excluded { + return Err(Gate::ExcludedPath); + } + Ok(request) + } + + /// Whether `host` is the claimed domain or its `www.` name. + pub(crate) fn claims_host(&self, host: &str) -> bool { + host.strip_prefix("www.").unwrap_or(host) == self.config.domain + } + + /// Find the article and its signable text, with no I/O. + fn extract(&self, html: &str) -> Result { + let boundary = boundary::find_article_boundary(html, Some(&self.config.article_selector)) + .ok_or(Reason::NoBoundary)?; + let article = html + .get(boundary.start..boundary.end) + .ok_or(Reason::NoBoundary)?; + if embed::has_existing_markers(article) { + return Err(Reason::AlreadySigned); + } + let metadata = text::document_metadata(html); + if metadata.manifest_present { + return Err(Reason::ManifestPresent); + } + let (visible, scalars) = text::visible_text(article, api::MAX_TEXT_SCALARS); + if scalars < self.config.min_text_length { + return Err(Reason::TextTooShort); + } + Ok(Article { + boundary, + text: visible, + scalars, + head_close: metadata.head_close, + title_range: metadata.title_range, + }) + } + + /// Unix time at which an entry written now with `ttl_seconds` expires. + fn expires_at(ttl_seconds: u64) -> u64 { + unix_now().saturating_add(ttl_seconds) + } + + /// Bound for skip writes made after resolve work stopped. + fn skip_deadline<'a>(services: &'a RuntimeServices) -> Deadline<'a> { + Deadline::new(services.timer(), SKIP_WRITE_RESERVE) + } + + /// Sign what a held document carries: the article's text first, as the + /// page path always has, then each embedded media item, within one + /// `max_hold_ms` budget. Returns the bytes to emit. + async fn resolve_document( + &self, + target: &Target<'_>, + services: &RuntimeServices, + document: &str, + article: Option
, + media: embedded::Scan, + ) -> Vec { + let request = target.request; + let mut work = Deadline::new( + services.timer(), + Duration::from_millis(self.config.max_hold_ms).saturating_sub(SKIP_WRITE_RESERVE), + ); + work.start().await; + let policy_digest = match self + .policy_digest(&request.page_url, services, services.kv_store(), &mut work) + .await + { + Ok(policy_digest) => policy_digest, + Err(reason) => { + if article.is_some() { + log_outcome(false, reason); + } + media::log_unsigned(&media, reason); + return document.as_bytes().to_vec(); + } + }; + let mut output = Cow::Borrowed(document); + if let Some(article) = &article { + match self + .resolve( + request, + services, + document, + article, + &policy_digest, + &mut work, + ) + .await + { + Ok((signed, reason)) => { + log_outcome(true, reason); + output = Cow::Owned(signed); + } + Err(reason) => log_outcome(false, reason), + } + } + if media.is_empty() { + return output.into_owned().into_bytes(); + } + // Markers go into text and the manifest meta tag into the head, never + // into an attribute, a style or an SVG, so a rescan finds the same + // media at its new offsets. + let media = match (&output, target.surface) { + (Cow::Owned(signed), Surface::Page) => embedded::scan_html(signed, target.policies), + _ => media, + }; + // The text plan may come from the cached policy, but media is signed + // or reused only under the policy the service answers now, so a + // revoked credential cannot reuse a cached media result. + let media_policy = match self + .request_policy(&request.page_url, services, &mut work) + .await + { + Ok(policy_digest) => policy_digest, + Err(reason) => { + media::log_unsigned(&media, reason); + return output.into_owned().into_bytes(); + } + }; + let embedded = media::EmbeddedJob { + request, + surface: target.surface, + policy_digest: &media_policy, + }; + match self + .sign_embedded(services, &embedded, &output, &media, &mut work) + .await + { + Some(signed) => signed.into_bytes(), + None => output.into_owned().into_bytes(), + } + } + + /// Resolve the plan for `article` under `policy_digest` and render it + /// into `html`. + async fn resolve( + &self, + request: &PageRequest, + services: &RuntimeServices, + html: &str, + article: &Article, + policy_digest: &str, + work: &mut Deadline<'_>, + ) -> Result<(String, Reason), Reason> { + let kv = services.kv_store(); + let suffix = format!( + "{}:{policy_digest}:{}", + self.cache_context, + text::content_hash_hex(&article.text) + ); + let plan_key = format!("{PLAN_KEY_PREFIX}:{suffix}"); + let skip_key = format!("{PLAN_SKIP_KEY_PREFIX}:{suffix}"); + let outcome = match self.read::(kv, &plan_key, work).await { + Ok(Some(CacheEntry::Plan { + plan, manifest_url, .. + })) => { + if api::validate_plan(&plan, manifest_url.as_deref(), article.scalars).is_err() { + self.delete_invalid_entry(kv, &plan_key, work).await; + return Err(Reason::CachedPlanInvalid); + } + return render( + html, + article, + &plan, + manifest_url.as_deref(), + self.max_document_bytes, + ) + .map(|signed| (signed, Reason::CacheHit)) + .map_err(render_reason); + } + Ok(Some(CacheEntry::Skip { .. })) => { + self.delete_invalid_entry(kv, &plan_key, work).await; + return Err(Reason::CachedEntryInvalid); + } + Err(Reason::Timeout) => Err(Reason::Timeout), + Err(reason) => return Err(reason), + Ok(None) => match self.read::(kv, &skip_key, work).await { + Ok(Some(CacheEntry::Skip { .. })) => return Err(Reason::CachedSkip), + Ok(Some(CacheEntry::Plan { .. })) => { + self.delete_invalid_entry(kv, &skip_key, work).await; + return Err(Reason::CachedEntryInvalid); + } + Err(Reason::Timeout) => Err(Reason::Timeout), + Err(reason) => return Err(reason), + Ok(None) => self + .request_plan(request, services, html, article, policy_digest, work) + .await + .and_then(|(plan, manifest_url)| { + api::validate_plan(&plan, manifest_url.as_deref(), article.scalars) + .map_err(|_| Reason::InvalidPlan)?; + let signed = render( + html, + article, + &plan, + manifest_url.as_deref(), + self.max_document_bytes, + ) + .map_err(render_reason)?; + Ok((signed, plan, manifest_url)) + }), + }, + }; + match outcome { + Ok((signed, plan, manifest_url)) => { + // A plan write that runs out of time is abandoned; the page is still signed. + let entry = CacheEntry::Plan { + plan, + manifest_url, + expires_at: Self::expires_at(self.config.plan_ttl_seconds), + ttl_less: false, + }; + self.write(kv, &plan_key, entry, self.config.plan_ttl_seconds, work) + .await; + Ok((signed, Reason::Signed)) + } + Err(reason) => { + let mut deadline = Self::skip_deadline(services); + if matches!(reason, Reason::PolicyChanged | Reason::PolicyDigestMismatch) { + self.evict_policy(kv, &mut deadline).await; + } + self.write_skip(kv, &skip_key, reason, &mut deadline).await; + Err(reason) + } + } + } + + async fn policy_digest( + &self, + page_url: &str, + services: &RuntimeServices, + kv: &dyn PlatformKvStore, + work: &mut Deadline<'_>, + ) -> Result { + match self.read::(kv, &self.policy_key, work).await { + Ok(Some(PolicyEntry::Digest { policy_digest, .. })) + if api::valid_policy_digest(&policy_digest) => + { + return Ok(policy_digest); + } + Ok(Some(PolicyEntry::Digest { .. })) | Ok(Some(PolicyEntry::Skip { .. })) => { + self.delete_invalid_entry(kv, &self.policy_key, work).await; + return Err(Reason::CachedEntryInvalid); + } + // Nothing can be cached without a store. + Err(Reason::KvUnavailable) => return Err(Reason::KvUnavailable), + Err(reason) => return Err(reason), + Ok(None) => {} + } + match self + .read::(kv, &self.policy_skip_key, work) + .await + { + Ok(Some(PolicyEntry::Skip { .. })) => return Err(Reason::CachedSkip), + Ok(Some(PolicyEntry::Digest { .. })) => { + self.delete_invalid_entry(kv, &self.policy_skip_key, work) + .await; + return Err(Reason::CachedEntryInvalid); + } + Err(Reason::KvUnavailable) => return Err(Reason::KvUnavailable), + Err(reason) => return Err(reason), + Ok(None) => {} + } + + let reason = match self.request_policy(page_url, services, work).await { + Ok(policy_digest) => { + let entry = PolicyEntry::Digest { + policy_digest: policy_digest.clone(), + expires_at: Self::expires_at(self.config.policy_ttl_seconds), + ttl_less: false, + }; + self.write( + kv, + &self.policy_key, + entry, + self.config.policy_ttl_seconds, + work, + ) + .await; + if !work.expired() { + return Ok(policy_digest); + } + // The digest is not cached, so the next view could not find a + // plan skip written under it; skip at the policy level instead. + Reason::Timeout + } + Err(reason) => reason, + }; + let ttl = self.policy_skip_ttl(); + let entry = PolicyEntry::Skip { + reason, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write( + kv, + &self.policy_skip_key, + entry, + ttl, + &mut Self::skip_deadline(services), + ) + .await; + Err(reason) + } + + fn policy_skip_ttl(&self) -> u64 { + self.config + .policy_ttl_seconds + .min(self.config.negative_ttl_seconds) + } + + async fn request_policy( + &self, + page_url: &str, + services: &RuntimeServices, + work: &mut Deadline<'_>, + ) -> Result { + let mut url = url::Url::parse(&self.policy_url).map_err(|_| Reason::RequestBuild)?; + url.query_pairs_mut().append_pair("page_url", page_url); + let body = self + .fetch( + services, + Method::GET, + url.as_str(), + EdgeBody::empty(), + api::MAX_POLICY_RESPONSE_BYTES, + work, + ) + .await?; + api::parse_policy_response(&body).map_err(|error| parse_error_reason(&error)) + } + + async fn evict_policy(&self, kv: &dyn PlatformKvStore, deadline: &mut Deadline<'_>) { + if !self.delete_cache_key(kv, &self.policy_key, deadline).await { + log::debug!("encypher: policy digest eviction failed"); + } + if !self + .delete_cache_key(kv, &self.policy_skip_key, deadline) + .await + { + log::debug!("encypher: policy skip eviction failed"); + } + } + + /// Call the sign API once. Every failure maps to a reason code. + async fn request_plan( + &self, + request: &PageRequest, + services: &RuntimeServices, + html: &str, + article: &Article, + policy_digest: &str, + work: &mut Deadline<'_>, + ) -> Result<(EmbeddingPlan, Option), Reason> { + let title = text::title_from_range(html, article.title_range); + let body = serde_json::to_vec(&SignRequest { + text: &article.text, + page_url: &request.page_url, + document_title: title.as_deref(), + boundary_selector: &article.boundary.selector, + expected_policy_digest: policy_digest, + }) + .map_err(|_| Reason::RequestBuild)?; + let body = self + .fetch( + services, + Method::POST, + &self.sign_url, + EdgeBody::from(body), + api::MAX_RESPONSE_BYTES, + work, + ) + .await?; + let parsed = api::parse_sign_response(&body).map_err(|error| parse_error_reason(&error))?; + if !parsed.success { + return Err(if parsed.error.as_deref() == Some("quota_exceeded") { + Reason::QuotaExceeded + } else { + Reason::SignRejected + }); + } + if parsed.policy_digest.as_deref() != Some(policy_digest) { + return Err(Reason::PolicyDigestMismatch); + } + let plan = parsed.embedding_plan.ok_or(Reason::NoPlan)?; + Ok((plan, parsed.manifest_url)) + } + + /// The worker token header value, marked sensitive so no log shows it. + fn worker_token(&self) -> Result { + let mut token = HeaderValue::from_str(self.config.worker_token.expose().trim()) + .map_err(|_| Reason::RequestBuild)?; + token.set_sensitive(true); + Ok(token) + } + + /// One bounded JSON HTTP call; never retried. + async fn fetch( + &self, + services: &RuntimeServices, + method: Method, + url: &str, + body: EdgeBody, + max_response_bytes: usize, + work: &mut Deadline<'_>, + ) -> Result { + let http_request = Request::builder() + .method(method) + .uri(url) + .header(header::CONTENT_TYPE, "application/json") + .header(WORKER_TOKEN_HEADER, self.worker_token()?) + .body(body) + .map_err(|_| Reason::RequestBuild)?; + let response_limit = max_response_bytes.saturating_add(1); + let response = self + .send_api(services, http_request, response_limit, work) + .await?; + let status = response.status(); + if status != StatusCode::OK { + return Err(match status.as_u16() { + 403 => Reason::Forbidden, + 409 => Reason::PolicyChanged, + 422 => Reason::Unprocessable, + 429 => Reason::RateLimited, + 500..=599 => Reason::ServerError, + _ => Reason::UnexpectedStatus, + }); + } + // One byte over the limit is enough for the parser to report TooLarge. + match work + .race(response.into_body().into_bytes_bounded(response_limit)) + .await + { + None => Err(Reason::Timeout), + Some(Err(_)) => Err(Reason::ResponseTooLarge), + Some(Ok(body)) => Ok(body), + } + } + + /// Send one request to the API backend, bounded by the first-byte + /// timeout and `work`, reading at most `response_limit` body bytes. + async fn send_api( + &self, + services: &RuntimeServices, + http_request: Request, + response_limit: usize, + work: &mut Deadline<'_>, + ) -> Result, Reason> { + let backend = ensure_integration_backend_with_timeout( + services, + &self.config.api_base_url, + INTEGRATION_ID, + Duration::from_millis(self.config.timeout_ms), + ) + .map_err(|_| Reason::Backend)?; + let platform_request = PlatformHttpRequest::new(http_request, backend) + .with_cache_bypass() + .with_max_response_body_bytes(response_limit); + match work + .race(services.http_client().send(platform_request)) + .await + { + None => Err(Reason::Timeout), + Some(Err(_)) => Err(Reason::HttpError), + Some(Ok(response)) => Ok(response.response), + } + } + + /// Read a fresh entry. Expired entries are misses; in TTL-less write mode + /// they are also deleted, best effort, so the store cleans itself. + async fn read( + &self, + kv: &dyn PlatformKvStore, + key: &str, + deadline: &mut Deadline<'_>, + ) -> Result, Reason> { + let bytes = match deadline.race(kv.get_bytes(key)).await { + None => return Err(Reason::Timeout), + Some(Err(KvError::Unavailable)) => return Err(Reason::KvUnavailable), + Some(Err(_)) => return Err(Reason::KvError), + Some(Ok(None)) => return Ok(None), + Some(Ok(Some(bytes))) => bytes, + }; + let entry: T = match serde_json::from_slice(&bytes) { + Ok(entry) => entry, + Err(_) => { + self.delete_invalid_entry(kv, key, deadline).await; + return Err(Reason::CachedEntryInvalid); + } + }; + // An entry written without a TTL shows the mode to an instance that has + // not written yet, as on Spin, which instantiates the module per request. + if entry.ttl_less() { + self.ttl_less.store(true, Ordering::Relaxed); + } + if is_fresh(entry.expires_at(), unix_now()) { + return Ok(Some(entry)); + } + if self.ttl_less.load(Ordering::Relaxed) && !self.delete_cache_key(kv, key, deadline).await + { + log::debug!("encypher: expired cache entry could not be deleted"); + } + Ok(None) + } + + /// Best-effort cache deletion, bounded by the caller's existing deadline. + async fn delete_cache_key( + &self, + kv: &dyn PlatformKvStore, + key: &str, + deadline: &mut Deadline<'_>, + ) -> bool { + matches!(deadline.race(kv.delete(key)).await, Some(Ok(()))) + } + + /// Delete a malformed or structurally invalid entry without masking fail-open. + async fn delete_invalid_entry( + &self, + kv: &dyn PlatformKvStore, + key: &str, + deadline: &mut Deadline<'_>, + ) { + if !self.delete_cache_key(kv, key, deadline).await { + log::debug!("encypher: invalid cache entry could not be deleted"); + } + } + + /// Best-effort cache write; failures are logged and ignored. + /// + /// Writes carry a TTL until the store first reports TTL writes as + /// unsupported; from then on this instance writes without one, and marks + /// the entry so a later instance that reads it knows the mode too. + async fn write( + &self, + kv: &dyn PlatformKvStore, + key: &str, + mut entry: E, + ttl_seconds: u64, + deadline: &mut Deadline<'_>, + ) { + if !self.ttl_less.load(Ordering::Relaxed) { + let Some(value) = encode(&entry) else { + return; + }; + let ttl = Duration::from_secs(ttl_seconds); + match deadline.race(kv.put_bytes_with_ttl(key, value, ttl)).await { + Some(Ok(())) => return, + Some(Err(KvError::Unsupported { .. })) => { + self.ttl_less.store(true, Ordering::Relaxed); + log::info!( + "encypher: the KV store does not support TTL writes; entries expire by their expires_at" + ); + } + Some(Err(_)) => { + log::debug!("encypher: cache write failed"); + return; + } + None => { + log::debug!("encypher: cache write timed out"); + return; + } + } + } + entry.mark_ttl_less(); + let Some(value) = encode(&entry) else { + return; + }; + match deadline.race(kv.put_bytes(key, value)).await { + Some(Ok(())) => {} + Some(Err(_)) => log::debug!("encypher: cache write failed"), + None => log::debug!("encypher: cache write timed out"), + } + } + + /// Write a negative article entry under its separate skip key. + async fn write_skip( + &self, + kv: &dyn PlatformKvStore, + key: &str, + reason: Reason, + deadline: &mut Deadline<'_>, + ) { + let ttl = self.config.negative_ttl_seconds; + let entry = CacheEntry::Skip { + reason, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write(kv, key, entry, ttl, deadline).await; + } +} + +/// Serialize a cache entry; a failure is logged and the write skipped. +fn encode(entry: &impl Serialize) -> Option { + match serde_json::to_vec(entry) { + Ok(value) => Some(Bytes::from(value)), + Err(_) => { + log::debug!("encypher: cache entry could not be serialized"); + None + } + } +} + +/// Apply a validated plan and optional manifest in one exact-sized output. +fn render( + html: &str, + article: &Article, + plan: &EmbeddingPlan, + manifest_url: Option<&str>, + max_output_bytes: usize, +) -> Result { + let manifest = manifest_url + .filter(|url| !url.is_empty()) + .and_then(|url| article.head_close.map(|offset| (url, offset))); + embed::apply_plan(html, &article.boundary, plan, manifest, max_output_bytes) +} + +fn render_reason(error: embed::ApplyError) -> Reason { + match error { + embed::ApplyError::Alignment => Reason::AlignmentFailed, + embed::ApplyError::TooLarge => Reason::OutputTooLarge, + } +} + +fn parse_error_reason(error: &ApiError) -> Reason { + match error { + ApiError::TooLarge => Reason::ResponseTooLarge, + ApiError::Malformed => Reason::MalformedResponse, + } +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use std::collections::HashMap; + use std::sync::atomic::AtomicUsize; + use std::sync::{LazyLock, Mutex}; + use std::task::Poll; + + use error_stack::Report; + use serde_json::json; + + use super::*; + use trusted_server_core::platform::test_support::{ + StoppedTimer, StubHttpClient, build_services_with_http_client, + }; + use trusted_server_core::platform::{ + PlatformCacheIntent, PlatformError, PlatformHttpClient, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, + }; + + const TOKEN: &str = "cwk_test_worker_token_value"; + const PAGE_URL: &str = "https://www.publisher.example.com/news/harbor-vote"; + const MARKER: &str = "\u{E0100}\u{E0101}\u{FE01}"; + const MANIFEST_URL: &str = "https://api.example.com/manifests/doc-1"; + const POLICY_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const ARTICLE: &str = "Harbor Council Votes\ +

Harbor council votes on the ferry schedule

\ +

The harbor council met on Tuesday evening and approved a revised ferry schedule \ + for the coming season.

Residents asked for earlier departures on weekdays.

\ +
Footer
"; + /// Simulated time that passes each time a [`TickTimer`] sleep is polled. + const TICK: Duration = Duration::from_millis(100); + + // --- test doubles ------------------------------------------------------- + + /// Return `Pending` `times` times, waking the task each time, then `Ready`. + async fn yield_pending(times: usize) { + for _ in 0..times { + let mut yielded = false; + futures::future::poll_fn(|cx| { + if yielded { + Poll::Ready(()) + } else { + yielded = true; + cx.waker().wake_by_ref(); + Poll::Pending + } + }) + .await; + } + } + + /// A timer on simulated time: each poll of a sleep advances it by [`TICK`]. + /// Sleeps only advance while the work they race is pending, as on a real + /// runtime where the clock runs while I/O is outstanding. + #[derive(Default)] + struct TickTimer { + requested: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl PlatformTimer for TickTimer { + async fn sleep(&self, duration: Duration) { + self.requested + .lock() + .expect("should lock requested durations") + .push(duration); + let ticks = duration.as_millis().div_ceil(TICK.as_millis()); + yield_pending(usize::try_from(ticks).expect("ticks should fit")).await; + } + } + + /// In-memory KV store that counts calls and can fail, stall or refuse TTLs. + #[derive(Default)] + struct MemoryKv { + entries: Mutex>, + ttls: Mutex>, + deletes: Mutex>, + reads: AtomicUsize, + ttl_writes: AtomicUsize, + /// Plan-key reads that report a miss before reads see the stored entry. + hidden_reads: AtomicUsize, + /// Every read fails with [`KvError::Unavailable`], as an unbound store does. + unavailable: bool, + /// Every read fails with an error other than unavailability. + fail_reads: bool, + /// Every read never completes. + hung_reads: bool, + /// Writes to keys with this prefix never complete. + hung_writes_to: Option<&'static str>, + /// The first write to this exact key never completes; later writes succeed. + hung_first_write_to: Option, + hung_first_write_taken: AtomicBool, + /// TTL writes fail with [`KvError::Unsupported`], as on Spin. + ttl_unsupported: bool, + /// `Pending` polls before every operation completes. + pending_polls: usize, + /// Positive entry inserted after a negative writer's final read but before its put. + concurrent_positive_before_skip_write: Mutex>, + } + + impl MemoryKv { + fn entries(&self) -> HashMap { + self.entries.lock().expect("should lock entries").clone() + } + + fn entry(&self, key: &str) -> serde_json::Value { + serde_json::from_slice(&self.entries()[key]).expect("should store JSON") + } + + fn insert(&self, key: &str, value: &serde_json::Value) { + self.entries.lock().expect("should lock entries").insert( + key.to_owned(), + Bytes::from(serde_json::to_vec(value).expect("should serialize")), + ); + } + + fn insert_raw(&self, key: &str, value: impl Into) { + self.entries + .lock() + .expect("should lock entries") + .insert(key.to_owned(), value.into()); + } + + fn entry_bytes(&self, key: &str) -> Bytes { + self.entries()[key].clone() + } + + fn remove(&self, key: &str) { + self.entries + .lock() + .expect("should lock entries") + .remove(key); + } + + /// The single article entry under either current article namespace. + fn only_entry(&self) -> (String, serde_json::Value) { + let entries: Vec<_> = self + .entries() + .into_keys() + .filter(|key| { + key.starts_with(PLAN_KEY_PREFIX) || key.starts_with(PLAN_SKIP_KEY_PREFIX) + }) + .collect(); + assert_eq!(entries.len(), 1, "should hold exactly one article entry"); + let key = entries.into_iter().next().expect("should have an entry"); + let value = self.entry(&key); + (key, value) + } + + fn reads(&self) -> usize { + self.reads.load(Ordering::SeqCst) + } + + async fn store(&self, key: &str, value: Bytes) -> Result<(), KvError> { + yield_pending(self.pending_polls).await; + if self + .hung_writes_to + .is_some_and(|prefix| key.starts_with(prefix)) + || (self.hung_first_write_to.as_deref() == Some(key) + && !self.hung_first_write_taken.swap(true, Ordering::SeqCst)) + { + futures::future::pending::<()>().await; + } + let writes_skip = serde_json::from_slice::(&value) + .ok() + .and_then(|entry| { + entry + .get("kind") + .and_then(|kind| kind.as_str()) + .map(str::to_owned) + }) + .as_deref() + == Some("skip"); + if writes_skip + && let Some((positive_key, positive)) = self + .concurrent_positive_before_skip_write + .lock() + .expect("should lock concurrent write") + .take() + { + self.entries + .lock() + .expect("should lock entries") + .insert(positive_key, positive); + } + self.entries + .lock() + .expect("should lock entries") + .insert(key.to_owned(), value); + Ok(()) + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + self.reads.fetch_add(1, Ordering::SeqCst); + yield_pending(self.pending_polls).await; + if self.hung_reads { + futures::future::pending::<()>().await; + } + if self.unavailable { + return Err(KvError::Unavailable); + } + if self.fail_reads { + return Err(KvError::LimitExceeded { + message: "test failure".to_owned(), + }); + } + if key.starts_with(PLAN_KEY_PREFIX) + && self + .hidden_reads + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + return Ok(None); + } + Ok(self + .entries + .lock() + .expect("should lock entries") + .get(key) + .cloned()) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.store(key, value).await + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + self.ttl_writes.fetch_add(1, Ordering::SeqCst); + if self.ttl_unsupported { + return Err(KvError::Unsupported { + operation: "put_bytes_with_ttl".to_owned(), + }); + } + self.ttls + .lock() + .expect("should lock TTLs") + .insert(key.to_owned(), ttl); + self.store(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + yield_pending(self.pending_polls).await; + self.deletes + .lock() + .expect("should lock deletes") + .push(key.to_owned()); + self.remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(edgezero_core::key_value_store::KvPage::default()) + } + } + + /// HTTP client whose send never completes. + #[derive(Default)] + struct HungHttpClient { + sends: AtomicUsize, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for HungHttpClient { + async fn send( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + self.sends.fetch_add(1, Ordering::SeqCst); + futures::future::pending().await + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + } + + /// Wraps [`StubHttpClient`] so the send and the response body each return + /// `Pending` before they are ready, as on Cloudflare, Spin and Axum. + struct YieldingHttpClient { + inner: Arc, + pending_polls: usize, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for YieldingHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + yield_pending(self.pending_polls).await; + let response = self.inner.send(request).await?; + let (parts, body) = response.response.into_parts(); + let bytes = body + .into_bytes_bounded(usize::MAX) + .await + .expect("should read the stub body"); + let polls = self.pending_polls; + let body = EdgeBody::stream(futures::stream::once(async move { + yield_pending(polls).await; + bytes + })); + Ok(PlatformResponse::new(http::Response::from_parts( + parts, body, + ))) + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + } + + /// Records log lines, with their level, emitted on the current thread. + struct ThreadLogger; + + thread_local! { + static LOGS: RefCell> = const { RefCell::new(Vec::new()) }; + } + + impl log::Log for ThreadLogger { + fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool { + true + } + + fn log(&self, record: &log::Record<'_>) { + LOGS.with(|logs| { + logs.borrow_mut() + .push(format!("{} {}", record.level(), record.args())); + }); + } + + fn flush(&self) {} + } + + static LOGGER: ThreadLogger = ThreadLogger; + + /// A sign-failure case: name and the API response, `None` for a transport error. + type FailureCase = (&'static str, Option<(u16, Vec)>); + /// A gate case: name, signer, and the request mutation that trips the gate. + type GateCase<'a> = (&'static str, &'a Arc, fn(&mut PageRequest)); + + /// Runs `run` with this thread's log lines captured. Returns `None`, after + /// skipping `run`, when another logger already owns the process. + fn capture_logs(run: impl FnOnce()) -> Option> { + static INSTALLED: LazyLock = LazyLock::new(|| { + let installed = log::set_logger(&LOGGER).is_ok(); + if installed { + log::set_max_level(log::LevelFilter::Trace); + } + installed + }); + if !*INSTALLED { + return None; + } + LOGS.with(|logs| logs.borrow_mut().clear()); + run(); + Some(LOGS.with(RefCell::take)) + } + + // --- fixtures ----------------------------------------------------------- + + fn config_json(token: &str) -> serde_json::Value { + json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": token, + }) + } + + fn factory_with(config: serde_json::Value, max_document_bytes: usize) -> Arc { + let config: EncypherConfig = serde_json::from_value(config).expect("should parse config"); + Arc::new(Signer::new(config, max_document_bytes)) + } + + fn factory() -> Arc { + factory_with(config_json(TOKEN), 5_000_000) + } + + fn article_with_selector(html: &str, selector: Option<&str>) -> Article { + let boundary = + boundary::find_article_boundary(html, selector).expect("should find boundary"); + let metadata = text::document_metadata(html); + let boundary_html = html + .get(boundary.start..boundary.end) + .expect("boundary should be valid"); + let (text, scalars) = text::visible_text(boundary_html, api::MAX_TEXT_SCALARS); + Article { + boundary, + text, + scalars, + head_close: metadata.head_close, + title_range: metadata.title_range, + } + } + + fn policy_key() -> String { + factory().policy_key.clone() + } + + fn policy_skip_key() -> String { + factory().policy_skip_key.clone() + } + + /// Services without a timer, as on Fastly. + fn services(http: Arc, kv: &Arc) -> RuntimeServices { + build_services_with_http_client(http) + .with_kv_store(Arc::clone(kv) as Arc) + } + + /// Services with a [`TickTimer`], as on Cloudflare, Spin and Axum. + fn timed_services(http: Arc, kv: &Arc) -> RuntimeServices { + services(http, kv).with_timer(Arc::new(TickTimer::default())) + } + + fn request() -> PageRequest { + PageRequest { + method: Method::GET, + page_url: PAGE_URL.to_owned(), + removed_prefix: String::new(), + response_status: 200, + origin_response_is_private: false, + } + } + + fn plan_json(marker: &str) -> serde_json::Value { + json!({ + "index_unit": "codepoint", + "operations": [{ "insert_after_index": 5, "marker": marker }], + }) + } + + fn sign_response(marker: &str) -> Vec { + sign_response_for_policy(marker, POLICY_DIGEST) + } + + fn policy_response(digest: &str) -> Vec { + serde_json::to_vec(&json!({"policy_digest": digest})).expect("should serialize policy") + } + + fn sign_response_for_policy(marker: &str, digest: &str) -> Vec { + serde_json::to_vec(&json!({ + "success": true, + "policy_digest": digest, + "embedding_plan": plan_json(marker), + "manifest_url": MANIFEST_URL, + "verification_url": "https://api.example.com/verify/doc-1", + "document_id": "doc-1", + })) + .expect("should serialize sign response") + } + + fn signing_http() -> Arc { + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + http.push_response(200, sign_response(MARKER)); + http + } + + fn seed_plan(kv: &Arc) -> String { + view(&factory(), &services(signing_http(), kv)) + } + + /// Drive one document the way the middleware and the publisher driver do: + /// the gates on the request, when there is one, then two chunks, then the + /// asynchronous finish with the core backstop. A request the gates refuse, + /// or none, holds nothing, so the document passes through. + fn run( + factory: &Arc, + request: Option, + services: &RuntimeServices, + html: &str, + ) -> String { + let Some(request) = request.and_then(|request| factory.admit(request).ok()) else { + return html.to_owned(); + }; + futures::executor::block_on(async { + let mut processor = document_processor(factory, request, None); + let (first, second) = html.as_bytes().split_at(html.len() / 2); + let mut output = processor + .process_chunk(first, false) + .expect("should process first chunk"); + output.extend( + processor + .finish(second, services) + .await + .expect("should finish the document"), + ); + String::from_utf8(output).expect("should emit UTF-8") + }) + } + + /// One view of [`ARTICLE`] at [`PAGE_URL`]. + fn view(factory: &Arc, services: &RuntimeServices) -> String { + run(factory, Some(request()), services, ARTICLE) + } + + fn assert_signed(output: &str) { + assert!(output.contains(MARKER), "should embed the plan marker"); + assert!( + output.contains("c2pa-manifest-url") && output.contains(MANIFEST_URL), + "should inject the manifest meta tag" + ); + assert_eq!( + output.matches(MARKER).count(), + 1, + "should add the marker once" + ); + } + + /// Rewrite the `expires_at` of the entry at `key` to `expires_at`. + fn set_expiry(kv: &MemoryKv, key: &str, expires_at: u64) { + let mut entry = kv.entry(key); + entry["expires_at"] = json!(expires_at); + kv.insert(key, &entry); + } + + fn expired() -> u64 { + unix_now().saturating_sub(1) + } + + // --- policy ------------------------------------------------------------- + + #[test] + fn policy_failures_are_negatively_cached_without_using_an_old_plan() { + let cases: Vec = vec![ + ("unavailable", Some((503, Vec::new()))), + ("forbidden", Some((403, Vec::new()))), + ("transport", None), + ("malformed", Some((200, b"not json".to_vec()))), + ("missing digest", Some((200, b"{}".to_vec()))), + ( + "uppercase", + Some((200, policy_response(&POLICY_DIGEST.to_uppercase()))), + ), + ( + "oversized", + Some((200, vec![b' '; api::MAX_POLICY_RESPONSE_BYTES + 2])), + ), + ]; + for (case, response) in cases { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + kv.remove(&policy_key()); + let http = Arc::new(StubHttpClient::new()); + if let Some((status, body)) = response { + http.push_response(status, body); + } + let services = services(Arc::clone(&http) as Arc, &kv); + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE, "{case}"); + } + assert_eq!(http.recorded_request_methods(), ["GET"], "{case}"); + assert_eq!(kv.entry(&policy_skip_key())["kind"], "skip", "{case}"); + assert_eq!( + kv.ttls.lock().expect("lock")[&policy_skip_key()], + Duration::from_secs(300) + ); + // Once the skip is gone, refresh may reuse only the matching old plan. + kv.remove(&policy_skip_key()); + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["GET", "GET"]); + } + } + + #[test] + fn malformed_cached_policy_fails_closed_for_one_view_then_recovers() { + let future = unix_now() + 300; + for value in [ + json!("not an entry"), + json!({"kind": "digest", "policy_digest": null, "expires_at": future}), + json!({"kind": "digest", "policy_digest": "bad", "expires_at": future}), + json!({ + "kind": "digest", "policy_digest": POLICY_DIGEST, "expires_at": future, + "extra": true, + }), + // `expires_at` is required. + json!({"kind": "digest", "policy_digest": POLICY_DIGEST}), + ] { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let key = policy_key(); + kv.insert(&key, &value); + let http = Arc::new(StubHttpClient::new()); + let services = services(Arc::clone(&http) as Arc, &kv); + + assert_eq!(view(&factory(), &services), ARTICLE, "{value}"); + assert!(http.recorded_request_methods().is_empty(), "{value}"); + assert!( + kv.deletes.lock().expect("lock deletes").contains(&key), + "{value}: malformed policy should be evicted" + ); + assert!( + !kv.entries().contains_key(&key), + "{value}: malformed policy should not remain" + ); + + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["GET"], "{value}"); + } + } + + #[test] + fn policy_cache_hit_and_article_or_policy_changes_select_the_correct_plan() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, sign_response(MARKER)); + let services = services(Arc::clone(&http) as Arc, &kv); + let changed = ARTICLE.replace("Tuesday", "Wednesday"); + assert_signed(&run(&factory(), Some(request()), &services, &changed)); + assert_eq!(http.recorded_request_methods(), ["POST"]); + + kv.remove(&policy_key()); + let next_digest = "b".repeat(64); + http.push_response(200, policy_response(&next_digest)); + http.push_response(200, sign_response_for_policy(MARKER, &next_digest)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["POST", "GET", "POST"]); + let count = http.recorded_request_methods().len(); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods().len(), count); + // Returning to the original policy selects its original plan, not the intervening one. + kv.remove(&policy_key()); + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_signed(&view(&factory(), &services)); + assert_eq!( + http.recorded_request_methods(), + ["POST", "GET", "POST", "GET"] + ); + } + + #[test] + fn conflict_or_response_digest_mismatch_evicts_policy_without_retrying() { + let mut missing: serde_json::Value = + serde_json::from_slice(&sign_response(MARKER)).expect("JSON"); + missing + .as_object_mut() + .expect("object") + .remove("policy_digest"); + for (status, body) in [ + ( + 409, + br#"{"detail":"TRUSTED_SERVER_POLICY_CHANGED"}"#.to_vec(), + ), + (409, b"unparseable conflict".to_vec()), + (200, sign_response_for_policy(MARKER, &"b".repeat(64))), + (200, sign_response_for_policy(MARKER, "invalid")), + (200, serde_json::to_vec(&missing).expect("JSON")), + ] { + let kv = Arc::new(MemoryKv::default()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + http.push_response(status, body); + let services = services(Arc::clone(&http) as Arc, &kv); + assert_eq!(view(&factory(), &services), ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert!(!kv.entries().contains_key(&policy_key())); + assert_eq!(kv.only_entry().1["kind"], "skip"); + // Next request refetches policy, but the failed article remains negatively cached. + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_eq!(view(&factory(), &services), ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET", "POST", "GET"]); + } + } + + #[test] + fn entries_expire_at_their_ttl_and_the_store_ttl_matches() { + let kv = Arc::new(MemoryKv::default()); + let before = unix_now(); + seed_plan(&kv); + let after = unix_now(); + let ttls = kv.ttls.lock().expect("lock").clone(); + let (plan_key, plan) = kv.only_entry(); + for (key, entry, ttl) in [ + (policy_key(), kv.entry(&policy_key()), 300), + (plan_key, plan, 604_800), + ] { + assert_eq!(ttls[&key], Duration::from_secs(ttl)); + let expires_at = entry["expires_at"] + .as_u64() + .expect("should carry expires_at"); + assert!( + (before + ttl..=after + ttl).contains(&expires_at), + "{key} should expire {ttl}s after it was written" + ); + } + for (policy_ttl, negative_ttl, expected) in [(60, 300, 60), (300, 60, 60)] { + let kv = Arc::new(MemoryKv::default()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + let mut config = config_json(TOKEN); + config["policy_ttl_seconds"] = json!(policy_ttl); + config["negative_ttl_seconds"] = json!(negative_ttl); + let factory = factory_with(config, 5_000_000); + assert_eq!(view(&factory, &services(http, &kv)), ARTICLE); + assert_eq!( + kv.ttls.lock().expect("lock")[&factory.policy_skip_key], + Duration::from_secs(expected) + ); + } + } + + // --- sign path and cache ------------------------------------------------ + + #[test] + fn signs_on_a_miss_with_the_worker_token_and_caches_the_plan() { + let http = signing_http(); + let kv = Arc::new(MemoryKv::default()); + let services = services(Arc::clone(&http) as Arc, &kv); + + let output = view(&factory(), &services); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert_eq!( + http.recorded_cache_intents(), + [PlatformCacheIntent::Bypass, PlatformCacheIntent::Bypass], + "credential-bound policy and sign responses must bypass intermediary caches" + ); + assert_eq!( + http.recorded_request_uris(), + [ + format!( + "https://api.example.com/api/v1/public/cdn/trusted-server/policy?page_url={}", + url::form_urlencoded::byte_serialize(PAGE_URL.as_bytes()).collect::() + ), + "https://api.example.com/api/v1/public/cdn/sign".to_owned(), + ] + ); + let headers = &http.recorded_request_headers()[1]; + let header = |name: &str| { + headers + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + }; + assert_eq!(header("x-encypher-worker-token"), Some(TOKEN)); + assert_eq!(header("content-type"), Some("application/json")); + assert_eq!( + header("idempotency-key"), + None, + "should send no idempotency key" + ); + let body: serde_json::Value = serde_json::from_slice(&http.recorded_request_bodies()[1]) + .expect("should send a JSON body"); + assert_eq!(body["page_url"], PAGE_URL); + assert_eq!(body["expected_policy_digest"], POLICY_DIGEST); + assert_eq!(body["document_title"], "Harbor Council Votes"); + assert_eq!(body["boundary_selector"], "article"); + let sent_text = body["text"].as_str().expect("should send text"); + assert!( + sent_text.starts_with("Harbor council votes on the ferry schedule The harbor council"), + "should send the article's visible text, got {sent_text}" + ); + assert!(!sent_text.contains("Footer") && !sent_text.contains("Home")); + let (key, entry) = kv.only_entry(); + assert!( + key.starts_with("encypher:plan:v4:") && key.contains(POLICY_DIGEST), + "unexpected cache key {key}" + ); + assert!(!key.contains(TOKEN), "cache key must not contain the token"); + assert!( + policy_key().starts_with("encypher:policy:v3:"), + "unexpected policy key" + ); + assert_eq!(entry["kind"], "plan"); + assert_eq!(entry["plan"], plan_json(MARKER)); + } + + #[test] + fn a_fresh_plan_that_expands_past_the_document_budget_passes_through() { + let http = signing_http(); + let kv = Arc::new(MemoryKv::default()); + let signer = factory_with(config_json(TOKEN), ARTICLE.len()); + + let output = view( + &signer, + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!( + output, ARTICLE, + "the original document fits its exact budget" + ); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + } + + #[test] + fn a_cached_plan_that_expands_past_the_document_budget_passes_through() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let http = Arc::new(StubHttpClient::new()); + let signer = factory_with(config_json(TOKEN), ARTICLE.len()); + + let output = view( + &signer, + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!( + output, ARTICLE, + "the original document fits its exact budget" + ); + assert!( + http.recorded_request_methods().is_empty(), + "a cached plan should not call the API" + ); + } + + #[test] + fn render_budget_matches_the_exact_escaped_output_size() { + let article = factory().extract(ARTICLE).expect("should extract article"); + let plan: EmbeddingPlan = + serde_json::from_value(plan_json(MARKER)).expect("should build a valid plan"); + let manifest_url = r#"https://manifest.example/?a=1&b="x""#; + let expected = render(ARTICLE, &article, &plan, Some(manifest_url), usize::MAX) + .expect("unbounded render should succeed"); + assert!( + expected.contains("a=1&b="x"<y>"), + "fixture should exercise every escaped manifest expansion" + ); + + assert_eq!( + render(ARTICLE, &article, &plan, Some(manifest_url), expected.len(),) + .expect("the exact output budget should fit"), + expected + ); + assert_eq!( + render( + ARTICLE, + &article, + &plan, + Some(manifest_url), + expected.len() - 1, + ), + Err(embed::ApplyError::TooLarge), + "one byte below the exact output size must fail before rendering" + ); + } + + #[test] + fn render_orders_equal_head_offset_markers_before_manifest_meta() { + let html = "A"; + let article = article_with_selector(html, Some("head")); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: vec![ + api::PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_owned(), + }, + api::PlanOperation { + insert_after_index: 0, + marker: "\u{fe01}".to_owned(), + }, + ], + }; + + assert_eq!( + render( + html, + &article, + &plan, + Some(r#"https://example.com/?a=1&b="x""#), + usize::MAX, + ) + .expect("render should succeed"), + concat!( + "A\u{fe00}\u{fe01}", + r#""#, + "" + ) + ); + } + + #[test] + fn render_preserves_manifest_placement_around_malformed_head_closes() { + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: vec![api::PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_owned(), + }], + }; + let meta = r#""#; + for (html, expected) in [ + ( + "
A
", + format!("{meta}
A\u{fe00}
"), + ), + ( + "
AB
", + format!("
A\u{fe00}{meta}B
"), + ), + ] { + let article = article_with_selector(html, None); + assert_eq!( + render( + html, + &article, + &plan, + Some("https://manifest.example.com/m"), + usize::MAX, + ) + .expect("render should succeed"), + expected + ); + } + } + + #[test] + fn render_omits_manifest_for_an_empty_url_or_missing_head_close() { + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: vec![api::PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_owned(), + }], + }; + for (html, selector, manifest_url, expected) in [ + ( + "
A
", + None, + Some("https://manifest.example.com/m"), + "
A\u{fe00}
", + ), + ( + "A", + Some("head"), + Some(""), + "A\u{fe00}", + ), + ] { + let article = article_with_selector(html, selector); + assert_eq!( + render(html, &article, &plan, manifest_url, usize::MAX,) + .expect("render should succeed"), + expected + ); + } + } + + #[test] + fn a_cached_plan_signs_without_calling_the_api() { + let kv = Arc::new(MemoryKv::default()); + let first = seed_plan(&kv); + + let http = Arc::new(StubHttpClient::new()); + let second = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!( + second, first, + "a cache hit should serve the same signed page" + ); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + + #[test] + fn a_cached_skip_passes_through_without_calling_the_api() { + let kv = Arc::new(MemoryKv::default()); + let first_http = Arc::new(StubHttpClient::new()); + first_http.push_response(200, policy_response(POLICY_DIGEST)); + first_http.push_response(403, b"{}".to_vec()); + view(&factory(), &services(first_http, &kv)); + assert_eq!(kv.only_entry().1["kind"], "skip"); + + let http = Arc::new(StubHttpClient::new()); + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + + #[test] + fn a_delayed_skip_write_cannot_overwrite_a_concurrent_plan() { + let seeded = Arc::new(MemoryKv::default()); + seed_plan(&seeded); + let (plan_key, _) = seeded.only_entry(); + let policy_key = policy_key(); + let kv = Arc::new(MemoryKv { + entries: Mutex::new(HashMap::from([( + policy_key.clone(), + seeded.entry_bytes(&policy_key), + )])), + concurrent_positive_before_skip_write: Mutex::new(Some(( + plan_key, + seeded + .entries() + .into_iter() + .find(|(key, _)| key.starts_with(PLAN_KEY_PREFIX)) + .expect("seeded plan") + .1, + ))), + ..MemoryKv::default() + }); + let failed = Arc::new(StubHttpClient::new()); + failed.push_response(503, Vec::new()); + + assert_eq!( + view( + &factory(), + &services(Arc::clone(&failed) as Arc, &kv), + ), + ARTICLE, + "the failed request should pass through" + ); + assert_eq!(failed.recorded_request_methods(), ["POST"]); + let entries = kv.entries(); + assert!( + entries + .values() + .any(|value| serde_json::from_slice::(value) + .is_ok_and(|entry| entry["kind"] == "plan")), + "the concurrent plan must survive" + ); + assert!( + entries + .values() + .any(|value| serde_json::from_slice::(value) + .is_ok_and(|entry| entry["kind"] == "skip")), + "the delayed failure should remain negatively cached under its own key" + ); + + let no_api = Arc::new(StubHttpClient::new()); + assert_signed(&view( + &factory(), + &services(Arc::clone(&no_api) as Arc, &kv), + )); + assert!(no_api.recorded_request_methods().is_empty()); + } + + #[test] + fn a_delayed_policy_skip_cannot_overwrite_a_concurrent_digest() { + let seeded = Arc::new(MemoryKv::default()); + seed_plan(&seeded); + let policy_key = policy_key(); + let kv = Arc::new(MemoryKv { + concurrent_positive_before_skip_write: Mutex::new(Some(( + policy_key.clone(), + seeded.entry_bytes(&policy_key), + ))), + ..MemoryKv::default() + }); + let failed = Arc::new(StubHttpClient::new()); + failed.push_response(503, Vec::new()); + + assert_eq!( + view( + &factory(), + &services(Arc::clone(&failed) as Arc, &kv), + ), + ARTICLE, + ); + assert_eq!(failed.recorded_request_methods(), ["GET"]); + let entries = kv.entries(); + assert!( + entries + .values() + .any(|value| serde_json::from_slice::(value) + .is_ok_and(|entry| entry["kind"] == "digest")), + "the concurrent policy digest must survive" + ); + assert!( + entries + .values() + .any(|value| serde_json::from_slice::(value) + .is_ok_and(|entry| entry["kind"] == "skip")), + "the delayed policy failure should remain under its own key" + ); + + let sign = Arc::new(StubHttpClient::new()); + sign.push_response(200, sign_response(MARKER)); + assert_signed(&view( + &factory(), + &services(Arc::clone(&sign) as Arc, &kv), + )); + assert_eq!(sign.recorded_request_methods(), ["POST"]); + } + + #[test] + fn sign_failures_pass_through_and_write_a_skip() { + let oversized = { + let mut body = sign_response(MARKER); + body.resize(api::MAX_RESPONSE_BYTES + 2, b' '); + body + }; + let quota = serde_json::to_vec(&json!({ "success": false, "error": "quota_exceeded" })) + .expect("should serialize"); + let insecure_manifest = serde_json::to_vec(&json!({ + "success": true, + "policy_digest": POLICY_DIGEST, + "embedding_plan": plan_json(MARKER), + "manifest_url": "javascript:alert(1)", + })) + .expect("should serialize"); + let cases: Vec = vec![ + ("403", Some((403, b"{}".to_vec()))), + ("422", Some((422, b"{}".to_vec()))), + ("429", Some((429, b"{}".to_vec()))), + ("5xx", Some((502, Vec::new()))), + ("transport error or first-byte timeout", None), + ("oversized body", Some((200, oversized))), + ("malformed body", Some((200, b"not json".to_vec()))), + ( + "invalid marker", + Some((200, sign_response(""))), + ), + ("quota", Some((200, quota))), + ("non-https manifest url", Some((200, insecure_manifest))), + ( + "null plan", + Some(( + 200, + serde_json::to_vec(&json!({ + "success": true, "policy_digest": POLICY_DIGEST, "embedding_plan": null + })) + .expect("JSON"), + )), + ), + ]; + + for (case, response) in cases { + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + if let Some((status, body)) = response { + http.push_response(status, body); + } + let kv = Arc::new(MemoryKv::default()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!( + output, ARTICLE, + "{case}: should pass the page through unchanged" + ); + assert_eq!( + http.recorded_backend_names().len(), + 2, + "{case}: one policy call and one sign call" + ); + assert_eq!( + kv.only_entry().1["kind"], + "skip", + "{case}: should cache a skip" + ); + } + } + + #[test] + fn a_tampered_cached_plan_is_evicted_then_the_next_view_recovers() { + for (field, value) in [ + ("/plan/operations/0/marker", ""), + ("/manifest_url", "http://manifest.example.com/doc-1"), + ] { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let (key, mut entry) = kv.only_entry(); + *entry + .pointer_mut(field) + .expect("seeded entry should have the field") = json!(value); + kv.insert(&key, &entry); + + let http = Arc::new(StubHttpClient::new()); + let services = services(Arc::clone(&http) as Arc, &kv); + assert_eq!( + view(&factory(), &services), + ARTICLE, + "{field}: should pass through" + ); + assert!( + http.recorded_backend_names().is_empty(), + "{field}: should make no API call on the failure view" + ); + assert!( + kv.deletes.lock().expect("lock deletes").contains(&key), + "{field}: the invalid plan should be evicted" + ); + + http.push_response(200, sign_response(MARKER)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["POST"], "{field}"); + } + } + + #[test] + fn malformed_ttl_less_plan_is_evicted_then_the_next_view_recovers() { + let kv = Arc::new(MemoryKv { + ttl_unsupported: true, + ..MemoryKv::default() + }); + seed_plan(&kv); + let (key, _) = kv.only_entry(); + kv.insert_raw(&key, Bytes::from_static(b"not json")); + let http = Arc::new(StubHttpClient::new()); + let services = services(Arc::clone(&http) as Arc, &kv); + + assert_eq!(view(&factory(), &services), ARTICLE); + assert!(http.recorded_request_methods().is_empty()); + assert!( + kv.deletes.lock().expect("lock deletes").contains(&key), + "malformed entries must be deleted even when their ttl_less flag cannot be decoded" + ); + + http.push_response(200, sign_response(MARKER)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["POST"]); + } + + #[test] + fn representation_alignment_failure_does_not_evict_a_valid_cached_plan() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let (key, _) = kv.only_entry(); + let signer = factory(); + let mut article = signer.extract(ARTICLE).expect("should extract article"); + article.boundary.end = ARTICLE.len() + 1; + let http = Arc::new(StubHttpClient::new()); + let services = services(http, &kv); + + let result = futures::executor::block_on(async { + let mut work = Deadline::Unbounded; + signer + .resolve( + &request(), + &services, + ARTICLE, + &article, + POLICY_DIGEST, + &mut work, + ) + .await + }); + + assert_eq!( + result.expect_err("alignment should fail"), + Reason::AlignmentFailed + ); + assert!( + kv.entries().contains_key(&key), + "valid plan should remain cached" + ); + assert!( + !kv.deletes.lock().expect("lock deletes").contains(&key), + "representation-specific alignment must not evict a reusable plan" + ); + } + + #[test] + fn extraction_cap_counts_the_inter_fragment_separator() { + let leading = "a".repeat(api::MAX_TEXT_SCALARS - 1); + let html = format!( + "

{leading}

{}

", + "&".repeat(100_000) + ); + let article = factory().extract(&html).expect("should extract article"); + + assert_eq!(article.scalars, api::MAX_TEXT_SCALARS); + assert_eq!(article.text, format!("{leading} ")); + } + + #[test] + fn underlength_article_maps_a_valid_api_range_plan_to_alignment_failed() { + let html = "
A
"; + let article = article_with_selector(html, None); + assert_eq!( + html.get(article.boundary.start..article.boundary.end), + Some(html) + ); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: vec![api::PlanOperation { + insert_after_index: 1, + marker: MARKER.to_owned(), + }], + }; + assert!( + api::validate_plan(&plan, None, api::MAX_TEXT_SCALARS).is_ok(), + "the plan index is valid within the signing API range" + ); + + let reason = render(html, &article, &plan, None, usize::MAX) + .map_err(render_reason) + .expect_err("the article has no canonical scalar at index 1"); + assert_eq!(reason, Reason::AlignmentFailed); + } + + #[test] + fn a_changed_credential_misses_the_cache() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + + let http = signing_http(); + let output = view( + &factory_with(config_json("cwk_rotated_token"), 5_000_000), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert_eq!( + kv.entries().len(), + 4, + "each credential should have its own policy and article entries" + ); + } + + // --- asynchronous resolve and the hold deadline ------------------------- + + #[test] + fn calls_that_are_pending_before_ready_are_awaited_and_sign() { + let timers: [Option>; 3] = [ + None, + Some(Arc::new(StoppedTimer)), + Some(Arc::new(TickTimer::default())), + ]; + for timer in timers { + let label = if timer.is_some() { "timer" } else { "no timer" }; + let kv = Arc::new(MemoryKv { + pending_polls: 2, + ..MemoryKv::default() + }); + let stub = signing_http(); + let http = Arc::new(YieldingHttpClient { + inner: Arc::clone(&stub), + pending_polls: 2, + }); + let mut services = services(http, &kv); + if let Some(timer) = timer { + services = services.with_timer(timer); + } + + let first = view(&factory(), &services); + let second = view(&factory(), &services); + + assert_signed(&first); + assert_eq!( + second, first, + "{label}: the second view should hit the cache" + ); + assert_eq!(stub.recorded_request_methods(), ["GET", "POST"], "{label}"); + assert_eq!(kv.only_entry().1["kind"], "plan", "{label}"); + } + } + + #[test] + fn a_hung_policy_call_times_out_writes_a_skip_and_the_next_view_makes_no_call() { + let kv = Arc::new(MemoryKv::default()); + let hung = Arc::new(HungHttpClient::default()); + + let output = view( + &factory(), + &timed_services(Arc::clone(&hung) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "should release the page unchanged"); + assert_eq!(hung.sends.load(Ordering::SeqCst), 1); + let skip = kv.entry(&policy_skip_key()); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + + let http = Arc::new(StubHttpClient::new()); + let next = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + assert_eq!(next, ARTICLE); + assert!( + http.recorded_backend_names().is_empty(), + "the next view should be a cached skip" + ); + } + + #[test] + fn a_hung_sign_call_or_body_times_out_writes_a_plan_skip_and_the_next_view_makes_no_call() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let article_key = kv.only_entry().0; + kv.remove(&article_key); + let hung = Arc::new(HungHttpClient::default()); + let services = timed_services(Arc::clone(&hung) as Arc, &kv); + + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE); + } + + assert_eq!( + hung.sends.load(Ordering::SeqCst), + 1, + "one sign call in total" + ); + let (_, skip) = kv.only_entry(); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + } + + #[test] + fn a_hung_kv_read_times_out_without_calling_the_api() { + let kv = Arc::new(MemoryKv { + hung_reads: true, + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 1); + assert!(http.recorded_backend_names().is_empty()); + } + + #[test] + fn a_plan_write_that_runs_out_of_time_still_returns_the_signed_page() { + let kv = Arc::new(MemoryKv { + hung_writes_to: Some(PLAN_KEY_PREFIX), + ..MemoryKv::default() + }); + let http = signing_http(); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert!( + kv.entries() + .keys() + .all(|key| !key.starts_with(PLAN_KEY_PREFIX)), + "the abandoned plan write should store nothing" + ); + } + + #[test] + fn a_policy_digest_write_that_runs_out_of_time_caches_a_policy_skip() { + let kv = Arc::new(MemoryKv { + hung_first_write_to: Some(policy_key()), + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "should release the page unchanged"); + assert_eq!( + http.recorded_request_methods(), + ["GET"], + "no sign call after the digest write ran out of time" + ); + let skip = kv.entry(&policy_skip_key()); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + assert!( + kv.entries() + .keys() + .all(|key| !key.starts_with(PLAN_SKIP_KEY_PREFIX)), + "no plan skip under a digest the next view cannot know" + ); + + // A hung client turns any API call into a full hold, so zero sends + // after the positive and negative policy reads shows a cached skip. + let reads = kv.reads(); + let hung = Arc::new(HungHttpClient::default()); + let next = view( + &factory(), + &timed_services(Arc::clone(&hung) as Arc, &kv), + ); + assert_eq!(next, ARTICLE); + assert_eq!(hung.sends.load(Ordering::SeqCst), 0, "no API call"); + assert_eq!(kv.reads() - reads, 2, "positive policy miss, then skip hit"); + } + + #[test] + fn the_core_backstop_lies_beyond_the_module_hold_bound() { + let kv = Arc::new(MemoryKv::default()); + let timer = Arc::new(TickTimer::default()); + let services = services(Arc::new(HungHttpClient::default()), &kv) + .with_timer(Arc::clone(&timer) as Arc); + + assert_eq!(view(&factory(), &services), ARTICLE); + + let max_hold = Duration::from_millis(5_000); + let requested = timer.requested.lock().expect("lock").clone(); + let (backstop, module): (Vec<_>, Vec<_>) = + requested.into_iter().partition(|sleep| *sleep > max_hold); + assert_eq!(backstop.len(), 1, "one core backstop beyond max_hold_ms"); + // The skip-write reserve starts only after the work deadline, so the + // module's own sleeps run back to back. + assert!( + !module.is_empty() && module.iter().sum::() <= max_hold, + "module sleeps {module:?} should fit within max_hold_ms" + ); + assert_eq!( + kv.entry(&policy_skip_key())["reason"], + "timeout", + "the module deadline, not the backstop, should release the page" + ); + } + + // --- expiry, write mode and key namespaces ------------------------------ + + #[test] + fn expired_entries_are_misses() { + // (entry to expire, API calls the next view makes) + let cases: [(&str, &[&str]); 4] = [ + ("policy digest", &["GET"]), + ("policy skip", &["GET"]), + ("plan", &["POST"]), + ("plan skip", &["POST"]), + ]; + for (case, expected_calls) in cases { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let (plan_key, _) = kv.only_entry(); + let key = if case.starts_with("policy") { + policy_key() + } else { + plan_key + }; + if case.ends_with("skip") { + kv.insert( + &key, + &json!({"kind": "skip", "reason": "server_error", "expires_at": expired()}), + ); + } else { + set_expiry(&kv, &key, expired()); + } + let http = Arc::new(StubHttpClient::new()); + if case.starts_with("policy") { + http.push_response(200, policy_response(POLICY_DIGEST)); + } else { + http.push_response(200, sign_response(MARKER)); + } + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), expected_calls, "{case}"); + assert!( + kv.entry(&key)["expires_at"].as_u64() > Some(unix_now()), + "{case}: should be replaced by a fresh entry" + ); + } + } + + #[test] + fn an_expired_plan_does_not_block_a_skip_write() { + let seeded = Arc::new(MemoryKv::default()); + seed_plan(&seeded); + let kv = Arc::new(MemoryKv { + entries: Mutex::new(seeded.entries()), + hidden_reads: AtomicUsize::new(1), + ..MemoryKv::default() + }); + let (plan_key, _) = kv.only_entry(); + set_expiry(&kv, &plan_key, expired()); + let failed = Arc::new(StubHttpClient::new()); + failed.push_response(503, Vec::new()); + + assert_eq!( + view( + &factory(), + &services(Arc::clone(&failed) as Arc, &kv), + ), + ARTICLE + ); + assert_eq!(failed.recorded_request_methods(), ["POST"]); + + let no_api = Arc::new(StubHttpClient::new()); + assert_eq!( + view( + &factory(), + &services(Arc::clone(&no_api) as Arc, &kv), + ), + ARTICLE + ); + assert!( + no_api.recorded_request_methods().is_empty(), + "the fresh skip should suppress a second signing call" + ); + } + + #[test] + fn a_store_without_ttl_writes_switches_once_and_expires_by_value() { + let kv = Arc::new(MemoryKv { + ttl_unsupported: true, + ..MemoryKv::default() + }); + let factory = factory(); + let http = signing_http(); + let services = services(Arc::clone(&http) as Arc, &kv); + + let first = view(&factory, &services); + + assert_signed(&first); + assert_eq!( + kv.ttl_writes.load(Ordering::SeqCst), + 1, + "only the first write should try a TTL" + ); + let (plan_key, plan) = kv.only_entry(); + assert_eq!(plan["kind"], "plan"); + assert!(plan["expires_at"].as_u64() > Some(unix_now())); + assert_eq!(kv.entry(&policy_key())["kind"], "digest"); + assert_eq!(view(&factory, &services), first, "should hit the cache"); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + + // Expired entries are misses and are deleted when read. + set_expiry(&kv, &plan_key, expired()); + http.push_response(200, sign_response(MARKER)); + assert_signed(&view(&factory, &services)); + assert_eq!(*kv.deletes.lock().expect("lock"), [plan_key.as_str()]); + assert_eq!( + kv.entry(&plan_key)["kind"], + "plan", + "should store a new plan" + ); + } + + #[test] + fn a_fresh_instance_deletes_expired_entries_written_without_a_ttl() { + // Spin instantiates the module per request, so the instance that reads + // an expired entry has not written anything yet. + let kv = Arc::new(MemoryKv { + ttl_unsupported: true, + ..MemoryKv::default() + }); + view(&factory(), &services(signing_http(), &kv)); + set_expiry(&kv, &policy_key(), expired()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET"]); + assert_eq!(*kv.deletes.lock().expect("lock"), [policy_key()]); + } + + #[test] + fn expired_entries_are_not_deleted_on_a_store_with_ttls() { + let kv = Arc::new(MemoryKv::default()); + let http = signing_http(); + let services = services(Arc::clone(&http) as Arc, &kv); + view(&factory(), &services); + let (plan_key, _) = kv.only_entry(); + set_expiry(&kv, &plan_key, expired()); + http.push_response(200, sign_response(MARKER)); + + assert_signed(&view(&factory(), &services)); + + assert!(kv.deletes.lock().expect("lock").is_empty()); + } + + #[test] + fn entries_under_the_previous_key_versions_are_never_read() { + let current = Arc::new(MemoryKv::default()); + seed_plan(¤t); + // The same plan and policy, as the previous release stored them. + let kv = Arc::new(MemoryKv::default()); + for (key, value) in current.entries() { + let mut entry: serde_json::Value = + serde_json::from_slice(&value).expect("should store JSON"); + entry.as_object_mut().expect("object").remove("expires_at"); + let old_key = key + .replace("encypher:plan:v4:", "encypher:v3:") + .replace("encypher:policy:v3:", "encypher:policy:v2:"); + kv.insert(&old_key, &entry); + } + let old_entries = kv.entries(); + let http = signing_http(); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + let entries = kv.entries(); + assert_eq!(entries.len(), old_entries.len() * 2); + for (key, value) in old_entries { + assert_eq!(entries[&key], value, "{key} should be left untouched"); + } + } + + // --- KV failures -------------------------------------------------------- + + #[test] + fn kv_read_errors_pass_through_without_calling_the_api() { + for kv in [ + MemoryKv { + fail_reads: true, + ..MemoryKv::default() + }, + MemoryKv { + unavailable: true, + ..MemoryKv::default() + }, + ] { + let kv = Arc::new(kv); + let http = Arc::new(StubHttpClient::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 1); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + } + + // --- gates and skips ---------------------------------------------------- + + #[test] + fn gated_documents_pass_through_with_no_kv_or_api_io() { + let signer = factory(); + let cases: Vec<(GateCase<'_>, Gate)> = vec![ + ( + ("method", &signer, |r| r.method = Method::POST), + Gate::Method, + ), + ( + ("status", &signer, |r| r.response_status = 404), + Gate::Status, + ), + ( + ("origin private", &signer, |r| { + r.origin_response_is_private = true; + }), + Gate::OriginPrivate, + ), + ( + ("host mismatch", &signer, |r| { + r.page_url = "https://other.example.com/news/harbor-vote".to_owned(); + }), + Gate::HostMismatch, + ), + ( + ("excluded path", &signer, |r| { + r.page_url = "https://publisher.example.com/WP-Admin/post".to_owned(); + }), + Gate::ExcludedPath, + ), + ( + ("excluded path under an addressing prefix", &signer, |r| { + r.page_url = + "https://publisher.example.com/www.site.example/wp-admin/post".to_owned(); + r.removed_prefix = "/www.site.example".to_owned(); + }), + Gate::ExcludedPath, + ), + ( + ("asset suffix", &signer, |r| { + r.page_url = "https://publisher.example.com/feed.XML".to_owned(); + }), + Gate::ExcludedPath, + ), + ( + ("invalid page url", &signer, |r| { + r.page_url = "not a url".to_owned(); + }), + Gate::InvalidPageUrl, + ), + ]; + + for ((case, signer, mutate), gate) in cases { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + let services = services(Arc::clone(&http) as Arc, &kv); + let mut request = request(); + mutate(&mut request); + + assert_eq!( + signer.admit(request.clone()), + Err(gate), + "{case}: should be refused by its gate" + ); + let output = run(signer, Some(request), &services, ARTICLE); + + assert_eq!(output, ARTICLE, "{case}: should pass through"); + assert_eq!(kv.reads(), 0, "{case}: should read no KV"); + assert!( + http.recorded_backend_names().is_empty(), + "{case}: no API call" + ); + } + + let kv = Arc::new(MemoryKv::default()); + assert_eq!( + run( + &factory(), + None, + &services(Arc::new(StubHttpClient::new()), &kv), + ARTICLE + ), + ARTICLE, + "nothing prepared" + ); + } + + #[test] + fn a_bare_domain_page_is_signed() { + let kv = Arc::new(MemoryKv::default()); + let mut request = request(); + request.page_url = "https://publisher.example.com/news/harbor-vote".to_owned(); + + let output = run( + &factory(), + Some(request), + &services(signing_http(), &kv), + ARTICLE, + ); + + assert_signed(&output); + } + + #[test] + fn an_oversized_document_is_flushed_and_passed_through() { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + + let output = view( + &factory_with(config_json(TOKEN), 64), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 0); + assert!(http.recorded_backend_names().is_empty()); + } + + #[test] + fn already_signed_or_short_pages_are_not_sent() { + let magic = "\u{E01DF}\u{E01E0}\u{E01E1}\u{E01E2}"; + let marked = ARTICLE.replace("The harbor council", &format!("The harbor{magic} council")); + let with_meta = ARTICLE.replace( + "", + r#""#, + ); + let short = "

Too short.

"; + + for page in [marked.as_str(), with_meta.as_str(), short] { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + + let output = run( + &factory(), + Some(request()), + &services(Arc::clone(&http) as Arc, &kv), + page, + ); + + assert_eq!(output, page); + assert_eq!(kv.reads(), 0, "should read no KV"); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + } + + #[test] + fn logs_carry_outcomes_but_never_the_token() { + let kv = Arc::new(MemoryKv::default()); + let Some(logs) = capture_logs(|| { + view(&factory(), &services(signing_http(), &kv)); + view(&factory(), &services(Arc::new(StubHttpClient::new()), &kv)); + let failing = Arc::new(StubHttpClient::new()); + failing.push_response(403, b"{}".to_vec()); + view( + &factory(), + &services(failing, &Arc::new(MemoryKv::default())), + ); + }) else { + // Log lines go to a logger installed elsewhere and cannot be observed here. + return; + }; + + assert!( + logs.iter() + .any(|line| line.ends_with("outcome=signed reason=signed")), + "should log the signing outcome: {logs:?}" + ); + for line in &logs { + assert!(!line.contains(TOKEN), "log line leaks the token: {line}"); + assert!( + !line.contains("encypher:plan:v4:") && !line.contains("encypher:policy:v3:"), + "log line leaks the cache key: {line}" + ); + assert!( + !line.contains("ferry"), + "log line leaks article text: {line}" + ); + } + } +} diff --git a/crates/provenance/encypher/src/processor/media.rs b/crates/provenance/encypher/src/processor/media.rs new file mode 100644 index 0000000000..842acd75ec --- /dev/null +++ b/crates/provenance/encypher/src/processor/media.rs @@ -0,0 +1,893 @@ +//! Media signing through the Encypher media endpoint. +//! +//! One source's exact bytes are sent with the server policy the edge holds, +//! and the answer is kept only after [`media_api::verify_signed_response`] +//! accepts it. Results are cached in the runtime KV store under a namespace +//! that binds the credential context, the policy digest, the delivery +//! address, the source reference, its declared and verified type, its bytes' +//! digest and whether a `Link` can be delivered, so a changed policy, +//! credential or source never reuses another's signature. The replay key the +//! endpoint requires is derived from the same namespace, so a repeated call +//! is answered with the identical bytes. +//! +//! Two callers use it: core's binary delivery seam, through [`EncypherMedia`], +//! and the page and stylesheet processors for embedded `data:` URIs and +//! inline SVG, through [`Signer::sign_embedded`]. Core keeps the signed +//! representations it delivers under [`MediaSigner::namespace`], which the +//! module answers only after a policy check no older than +//! `policy_ttl_seconds`, so a revoked credential or changed policy stops +//! every kept copy from being served. + +use std::sync::Arc; +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use http::{HeaderMap, HeaderName, HeaderValue, Method, Request, StatusCode, header}; +use serde::{Deserialize, Serialize}; +use trusted_server_core::media_signing::{ + LargeMediaJob, LargeMediaSigning, MediaJob, MediaNamespacing, MediaSigner, MediaSigning, + MediaSigningFailure, SignedMedia as DeliveredMedia, +}; +use trusted_server_core::middleware::ResponseHeader; +use trusted_server_core::platform::RuntimeServices; + +use super::{ + Deadline, Expiring, PageRequest, Reason, SKIP_WRITE_RESERVE, Signer, Surface, + WORKER_TOKEN_HEADER, is_excluded_prefix, unix_now, +}; +use crate::MODULE; +use crate::data_uri::DataUri; +use crate::embedded::{self, Scan}; +use crate::media_api::{ + self, Disposition, Expected, MediaMetadata, ResponseError, SignedMedia, SourceMetadata, + canonical_media_type, safe_source_reference, sha256_hex, valid_origin_host, +}; +use crate::svg_equivalence; + +/// Verified signed media. Entries carry their own `expires_at`. +const MEDIA_KEY_PREFIX: &str = "encypher:media:v1"; +/// Media failures, separate so a delayed failure cannot replace a result. +const MEDIA_SKIP_KEY_PREFIX: &str = "encypher:media-skip:v1"; +/// A fresh replay-key salt, written when the endpoint no longer holds or +/// refuses the result for the previous key. +const MEDIA_REPLAY_KEY_PREFIX: &str = "encypher:media-replay:v1"; +const SVG_MEDIA_TYPE: &str = "image/svg+xml"; + +/// The media types the signer signs: images, audio, video and fonts, +/// whatever the path they are delivered on. Whether a format is signable is +/// decided by its verified bytes at the endpoint, which answers an +/// unsupported one `422`, delivered unsigned. +const SIGNABLE_TYPE_PREFIXES: &[&str] = &["image/", "audio/", "video/", "font/"]; + +/// The media type of one segment of a fragmented ISO BMFF stream, as ISO/IEC +/// 14496-12 registers it. A segment's provenance is bound through its +/// stream's init segment, as with C2PA verifiable segment info, so it is +/// never signed alone and reaches the reader as the origin sent it. +const MEDIA_SEGMENT_TYPE: &str = "video/iso.segment"; + +const INSTANCE_ID_HEADER: HeaderName = HeaderName::from_static("x-encypher-manifest-instance-id"); +const DOCUMENT_ID_HEADER: HeaderName = HeaderName::from_static("x-encypher-document-id"); +const VERIFICATION_URL_HEADER: HeaderName = HeaderName::from_static("x-encypher-verification-url"); + +/// One media source to sign, as it is delivered. +struct MediaSource<'a> { + /// The reader-facing address the media is delivered on or in. + page_url: &'a str, + /// A safe canonical address of the source, or none. + reference: Option<&'a str>, + /// The hostname the source was fetched from, or none for media embedded + /// in a document. + origin_host: Option<&'a str>, + /// The declared type, sent as a hint. + mime: &'a str, + /// The bare type the signed bytes must keep. + media_type: &'a str, + bytes: &'a [u8], + sha256: &'a str, + /// Whether every response of the representation can carry a `Link`. + link_deliverable: bool, + /// The bytes are the platform image optimizer's answer, so the endpoint + /// must refuse them unless they carry an intact embedded manifest, the + /// optimizer's own record of its input and what it did. + require_embedded_manifest: bool, +} + +/// A cached media result. +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum MediaEntry { + /// The verified answer: its base64 bytes and its signing headers, which + /// are verified again on every read. + Signed { + bytes: String, + headers: Vec<(String, String)>, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Skip { + reason: Reason, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Replay { + salt: String, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, +} + +impl Expiring for MediaEntry { + fn expires_at(&self) -> u64 { + match self { + Self::Signed { expires_at, .. } + | Self::Skip { expires_at, .. } + | Self::Replay { expires_at, .. } => *expires_at, + } + } + + fn ttl_less(&self) -> bool { + match self { + Self::Signed { ttl_less, .. } + | Self::Skip { ttl_less, .. } + | Self::Replay { ttl_less, .. } => *ttl_less, + } + } + + fn mark_ttl_less(&mut self) { + match self { + Self::Signed { ttl_less, .. } + | Self::Skip { ttl_less, .. } + | Self::Replay { ttl_less, .. } => *ttl_less = true, + } + } +} + +impl MediaEntry { + fn signed(signed: &SignedMedia, headers: &HeaderMap, expires_at: u64) -> Self { + let headers = media_api::SIGNING_HEADERS + .iter() + .filter_map(|name| { + let value = headers.get(*name)?.to_str().ok()?; + Some(((*name).to_owned(), value.to_owned())) + }) + .collect(); + Self::Signed { + bytes: STANDARD.encode(&signed.bytes), + headers, + expires_at, + ttl_less: false, + } + } +} + +/// A cached answer, verified as if it had just arrived. +fn verify_cached( + bytes: &str, + headers: &[(String, String)], + expected: &Expected<'_>, +) -> Result { + let body = STANDARD + .decode(bytes) + .map_err(|_| Reason::CachedEntryInvalid)?; + let mut map = HeaderMap::with_capacity(headers.len()); + for (name, value) in headers { + let name = + HeaderName::from_bytes(name.as_bytes()).map_err(|_| Reason::CachedEntryInvalid)?; + let value = HeaderValue::from_str(value).map_err(|_| Reason::CachedEntryInvalid)?; + map.append(name, value); + } + media_api::verify_signed_response(&map, Bytes::from(body), expected) + .map_err(|_| Reason::CachedEntryInvalid) +} + +fn response_reason(error: ResponseError) -> Reason { + match error { + ResponseError::ContentType => Reason::SignedTypeMismatch, + ResponseError::PolicyDigest => Reason::PolicyDigestMismatch, + ResponseError::SignedDigest => Reason::SignedDigestMismatch, + ResponseError::Malformed => Reason::InvalidSignedResponse, + ResponseError::LinkUndeliverable => Reason::ManifestUndeliverable, + ResponseError::Unsigned => Reason::UnsignedResponse, + } +} + +/// The reason for an error answer, by its status and failure code. +fn status_reason(status: StatusCode, code: Option<&str>) -> Reason { + match status.as_u16() { + 403 | 404 => Reason::Forbidden, + 409 if code == Some("SIGNING_IDEMPOTENCY_CONFLICT") => Reason::IdempotencyConflict, + 409 => Reason::PolicyChanged, + 410 => Reason::ResultPurged, + 413 => Reason::PayloadTooLarge, + 422 => Reason::Unprocessable, + 429 => Reason::RateLimited, + 500..=599 => Reason::ServerError, + _ => Reason::UnexpectedStatus, + } +} + +fn log_media(signed: bool, reason: Reason) { + log::info!( + "encypher: media outcome={} reason={}", + if signed { "signed" } else { "unsigned" }, + reason.code() + ); +} + +/// Reports every media item of `scan` unsigned for `reason`. +pub(super) fn log_unsigned(scan: &Scan, reason: Reason) { + for refused in &scan.refused { + log_media(false, *refused); + } + for _ in 0..scan.found.len() + scan.svgs.len() { + log_media(false, reason); + } +} + +/// The document an embedded media pass works for. +pub(super) struct EmbeddedJob<'a> { + pub(super) request: &'a PageRequest, + pub(super) surface: Surface, + pub(super) policy_digest: &'a str, +} + +/// One item's outcome in an embedded pass: signed, from the cache or fresh, +/// or unsigned with why. +type Outcome = Result; + +impl Signer { + fn max_media_bytes(&self) -> usize { + self.config.max_media_bytes + } + + /// The cache namespace of `source` under `policy_digest`: every input the + /// signed result depends on. + fn media_namespace(&self, source: &MediaSource<'_>, policy_digest: &str) -> String { + sha256_hex( + format!( + "{}\n{policy_digest}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}", + self.cache_context, + source.page_url, + source.reference.unwrap_or_default(), + source.origin_host.unwrap_or_default(), + source.mime, + source.media_type, + source.sha256, + source.link_deliverable, + source.require_embedded_manifest + ) + .as_bytes(), + ) + } + + /// Sign one source, from the cache when a verified result is held. + /// Returns the result and whether it was signed now or read back. A call + /// to the endpoint is made only while `calls` is below + /// `self.config.max_media_sign_calls_per_document`, and counted there. + async fn sign_media( + &self, + services: &RuntimeServices, + source: &MediaSource<'_>, + policy_digest: &str, + work: &mut Deadline<'_>, + calls: &mut usize, + ) -> Result<(SignedMedia, Reason), Reason> { + if source.bytes.len() > self.max_media_bytes() { + return Err(Reason::MediaTooLarge); + } + // Once the budget is spent no call is made and no skip written, so + // the reserve stays for the document's own writes. + if work.expired() { + return Err(Reason::Timeout); + } + let kv = services.kv_store(); + let namespace = self.media_namespace(source, policy_digest); + let key = format!("{MEDIA_KEY_PREFIX}:{namespace}"); + let skip_key = format!("{MEDIA_SKIP_KEY_PREFIX}:{namespace}"); + let expected = Expected { + policy_digest, + media_type: source.media_type, + unsigned_sha256: source.sha256, + link_deliverable: source.link_deliverable, + }; + let outcome = match self.read::(kv, &key, work).await { + Ok(Some(MediaEntry::Signed { bytes, headers, .. })) => { + return match verify_cached(&bytes, &headers, &expected) { + Ok(signed) => Ok((signed, Reason::CacheHit)), + Err(reason) => { + self.delete_invalid_entry(kv, &key, work).await; + Err(reason) + } + }; + } + Ok(Some(_)) => { + self.delete_invalid_entry(kv, &key, work).await; + return Err(Reason::CachedEntryInvalid); + } + Err(Reason::Timeout) => Err(Reason::Timeout), + Err(reason) => return Err(reason), + Ok(None) => match self.read::(kv, &skip_key, work).await { + Ok(Some(MediaEntry::Skip { .. })) => return Err(Reason::CachedSkip), + Ok(Some(_)) => { + self.delete_invalid_entry(kv, &skip_key, work).await; + return Err(Reason::CachedEntryInvalid); + } + Err(Reason::Timeout) => Err(Reason::Timeout), + Err(reason) => return Err(reason), + Ok(None) if *calls >= self.config.max_media_sign_calls_per_document => { + return Err(Reason::Deferred); + } + Ok(None) => { + *calls += 1; + self.request_media(services, source, policy_digest, &namespace, &expected, work) + .await + } + }, + }; + match outcome { + Ok((signed, headers)) => { + // A result write that runs out of time is abandoned; the media + // is still signed. + let entry = MediaEntry::signed( + &signed, + &headers, + Self::expires_at(self.config.plan_ttl_seconds), + ); + self.write(kv, &key, entry, self.config.plan_ttl_seconds, work) + .await; + Ok((signed, Reason::Signed)) + } + Err(reason @ (Reason::ResultPurged | Reason::IdempotencyConflict)) => { + // The next view asks again under a new replay key; no skip, + // and never a retry within this view. + let entry = MediaEntry::Replay { + salt: format!("{}", unix_now()), + expires_at: Self::expires_at(self.config.plan_ttl_seconds), + ttl_less: false, + }; + let replay_key = format!("{MEDIA_REPLAY_KEY_PREFIX}:{namespace}"); + self.write( + kv, + &replay_key, + entry, + self.config.plan_ttl_seconds, + &mut Self::skip_deadline(services), + ) + .await; + Err(reason) + } + Err(reason) => { + let mut deadline = Self::skip_deadline(services); + if matches!(reason, Reason::PolicyChanged | Reason::PolicyDigestMismatch) { + self.evict_policy(kv, &mut deadline).await; + } + let ttl = self.config.negative_ttl_seconds; + let entry = MediaEntry::Skip { + reason, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write(kv, &skip_key, entry, ttl, &mut deadline).await; + Err(reason) + } + } + } + + /// Call the media endpoint once with the source's exact bytes. Every + /// failure maps to a reason code. + async fn request_media( + &self, + services: &RuntimeServices, + source: &MediaSource<'_>, + policy_digest: &str, + namespace: &str, + expected: &Expected<'_>, + work: &mut Deadline<'_>, + ) -> Result<(SignedMedia, HeaderMap), Reason> { + let kv = services.kv_store(); + let replay_key = format!("{MEDIA_REPLAY_KEY_PREFIX}:{namespace}"); + let salt = match self.read::(kv, &replay_key, work).await { + Ok(Some(MediaEntry::Replay { salt, .. })) => salt, + Ok(Some(_)) => { + self.delete_invalid_entry(kv, &replay_key, work).await; + return Err(Reason::CachedEntryInvalid); + } + Ok(None) => String::new(), + Err(reason) => return Err(reason), + }; + let idempotency_key = sha256_hex(format!("{namespace}\n{salt}").as_bytes()); + let metadata = serde_json::to_vec(&MediaMetadata::unmodified( + source.page_url, + policy_digest, + SourceMetadata { + reference: source.reference, + origin_host: source.origin_host, + mime: source.mime, + sha256: source.sha256, + byte_length: source.bytes.len(), + require_embedded_manifest: source.require_embedded_manifest, + }, + source.link_deliverable, + )) + .map_err(|_| Reason::RequestBuild)?; + let multipart = media_api::multipart(&metadata, source.bytes); + if multipart.body.len() > media_api::MAX_REQUEST_BYTES { + return Err(Reason::PayloadTooLarge); + } + let http_request = Request::builder() + .method(Method::POST) + .uri(&self.media_sign_url) + .header(header::CONTENT_TYPE, multipart.content_type) + .header(WORKER_TOKEN_HEADER, self.worker_token()?) + .header(media_api::IDEMPOTENCY_KEY_HEADER, idempotency_key) + .body(EdgeBody::from_bytes(multipart.body)) + .map_err(|_| Reason::RequestBuild)?; + let max_body = source + .bytes + .len() + .saturating_add(media_api::MAX_MANIFEST_OVERHEAD_BYTES); + let response_limit = max_body.saturating_add(1); + let response = self + .send_api(services, http_request, response_limit, work) + .await?; + let status = response.status(); + let headers = response.headers().clone(); + if status != StatusCode::OK { + let code = work + .race( + response + .into_body() + .into_bytes_bounded(media_api::MAX_ERROR_BODY_BYTES), + ) + .await + .and_then(Result::ok) + .and_then(|body| media_api::error_code(&body)); + log::info!( + "encypher: media sign answered status={} code={}", + status.as_u16(), + code.as_deref().unwrap_or("none") + ); + return Err(status_reason(status, code.as_deref())); + } + let body = match work + .race(response.into_body().into_bytes_bounded(response_limit)) + .await + { + None => return Err(Reason::Timeout), + Some(Err(_)) => return Err(Reason::ResponseTooLarge), + Some(Ok(body)) => body, + }; + if body.len() > max_body { + return Err(Reason::ResponseTooLarge); + } + let signed = + media_api::verify_signed_response(&headers, body, expected).map_err(response_reason)?; + Ok((signed, headers)) + } + + /// Sign the editable media embedded in `document`: each `data:` URI + /// found, then each inline SVG last as the equivalent standalone document. + /// The scan excludes entire original provenance-bearing or uncertain SVG + /// ranges before either pass, so child signing cannot invalidate them. + /// Returns the new document, or `None` when nothing in it was signed. + pub(super) async fn sign_embedded( + &self, + services: &RuntimeServices, + job: &EmbeddedJob<'_>, + document: &str, + scan: &Scan, + work: &mut Deadline<'_>, + ) -> Option { + for reason in &scan.refused { + log_media(false, *reason); + } + let mut outcomes: Vec = Vec::new(); + let mut edits: Vec<(usize, usize, String)> = Vec::new(); + let mut signed_sources: Vec<(String, String, Result)> = Vec::new(); + let mut calls = 0_usize; + for found in &scan.found { + let Some(uri) = document + .get(found.start..found.end) + .and_then(DataUri::parse) + else { + continue; + }; + let Ok(bytes) = uri.decode() else { + outcomes.push(Err(Reason::MalformedDataUri)); + continue; + }; + let media_type = uri.media_type(); + let sha256 = sha256_hex(&bytes); + let cached = signed_sources + .iter() + .find(|(sha, kind, _)| *sha == sha256 && *kind == media_type) + .map(|(_, _, result)| result.clone()); + let result = match cached { + Some(result) => result.map(|signed| (signed, Reason::CacheHit)), + None => { + let source = MediaSource { + page_url: &job.request.page_url, + reference: None, + origin_host: None, + mime: uri.declared_type(), + media_type: &media_type, + bytes: &bytes, + sha256: &sha256, + link_deliverable: false, + require_embedded_manifest: false, + }; + let result = self + .sign_media(services, &source, job.policy_digest, work, &mut calls) + .await + .and_then(|(signed, reason)| { + if media_type == SVG_MEDIA_TYPE { + embedded::verify_svg_carrier(&bytes, &signed.bytes)?; + } + Ok((signed, reason)) + }); + signed_sources.push(( + sha256.clone(), + media_type.clone(), + result + .as_ref() + .map(|(signed, _)| signed.clone()) + .map_err(|reason| *reason), + )); + result + } + }; + match result { + Ok((signed, reason)) => { + edits.push((found.start, found.end, uri.encode(&signed.bytes))); + outcomes.push(Ok(reason)); + } + Err(reason) => outcomes.push(Err(reason)), + } + } + let mut output = None; + if !edits.is_empty() { + output = embedded::splice(document, &edits, self.max_document_bytes); + if output.is_none() { + revoke(&mut outcomes); + } + } + if job.surface == Surface::Page { + let current = output.as_deref().unwrap_or(document); + let svgs = if output.is_some() { + embedded::scan_html(current, None).svgs + } else { + scan.svgs.clone() + }; + let mut svg_outcomes = Vec::with_capacity(svgs.len()); + let mut svg_edits = Vec::new(); + for (start, end) in svgs { + let Some(fragment) = current.get(start..end) else { + continue; + }; + match self + .sign_inline_svg(services, job, fragment, work, &mut calls) + .await + { + Ok((signed, reason)) => { + svg_edits.push((start, end, signed)); + svg_outcomes.push(Ok(reason)); + } + Err(reason) => svg_outcomes.push(Err(reason)), + } + } + if !svg_edits.is_empty() { + match embedded::splice(current, &svg_edits, self.max_document_bytes) { + Some(signed) => output = Some(signed), + None => revoke(&mut svg_outcomes), + } + } + outcomes.extend(svg_outcomes); + } + for outcome in &outcomes { + match outcome { + Ok(reason) => log_media(true, *reason), + Err(reason) => log_media(false, *reason), + } + } + output + } + + /// Sign one inline SVG as the standalone document + /// [`signable_svg`](svg_equivalence::signable_svg) proves equivalent to + /// the browser's tree: the original bytes where they already are one, + /// else the browser's tree serialized as XML. Those bytes are hashed, + /// sent, cached and compared with the answer, and the result is kept + /// only when the signer added its manifest and changed nothing else. + async fn sign_inline_svg( + &self, + services: &RuntimeServices, + job: &EmbeddedJob<'_>, + fragment: &str, + work: &mut Deadline<'_>, + calls: &mut usize, + ) -> Result<(String, Reason), Reason> { + let svg = svg_equivalence::signable_svg(fragment, self.max_media_bytes())?; + let bytes = svg.as_bytes(); + let sha256 = sha256_hex(bytes); + let source = MediaSource { + page_url: &job.request.page_url, + reference: None, + origin_host: None, + mime: SVG_MEDIA_TYPE, + media_type: SVG_MEDIA_TYPE, + bytes, + sha256: &sha256, + link_deliverable: false, + require_embedded_manifest: false, + }; + let (signed, reason) = self + .sign_media(services, &source, job.policy_digest, work, calls) + .await?; + embedded::verify_svg_carrier(bytes, &signed.bytes)?; + let signed = + String::from_utf8(signed.bytes.to_vec()).map_err(|_| Reason::InlineSvgAltered)?; + Ok((signed, reason)) + } + + /// The namespace core keeps this credential's signed representations + /// under: the module version, the policy digest the service answers now, + /// and the credential context hash. Never a secret. The policy is asked + /// for the claim's root, being the policy of the credential whatever the + /// asset, and asked live on every call, never from the article path's + /// cache, so a revoked credential or changed policy stops every kept copy + /// at once. + async fn delivery_namespace(&self, services: &RuntimeServices) -> Result { + let mut work = Deadline::new( + services.timer(), + Duration::from_millis(self.config.max_hold_ms).saturating_sub(SKIP_WRITE_RESERVE), + ); + work.start().await; + let claim_root = format!("https://{}/", self.config.domain); + let policy_digest = self + .request_policy(&claim_root, services, &mut work) + .await?; + Ok(self.kept_namespace(&policy_digest)) + } + + /// The namespace of signed representations made under `policy_digest` + /// with this credential, as [`Self::delivery_namespace`] answers it. + pub(super) fn kept_namespace(&self, policy_digest: &str) -> String { + format!("{NAMESPACE_VERSION}:{policy_digest}:{}", self.cache_context) + } + + /// Sign one binary representation core captured, for every response of + /// it: the reader-facing address must be on the claimed domain, the + /// bytes the ones core hashed. + async fn sign_delivery( + &self, + services: &RuntimeServices, + job: &MediaJob<'_>, + ) -> Result { + let reader_url = url::Url::parse(job.reader_url).map_err(|_| Reason::OutsideClaim)?; + let host = reader_url.host_str().ok_or(Reason::OutsideClaim)?; + if !matches!(reader_url.scheme(), "https" | "http") + || !self.claims_host(host) + || is_excluded_prefix(reader_url.path()) + { + return Err(Reason::OutsideClaim); + } + if job.source.len() > self.max_media_bytes() { + return Err(Reason::MediaTooLarge); + } + let sha256 = sha256_hex(job.source); + if sha256 != job.source_sha256 { + return Err(Reason::SourceDigestMismatch); + } + // The host core fetched from, which the endpoint checks the claim's + // managed origins and the reference against. + let origin_host = job + .source_origin_host + .filter(|host| valid_origin_host(host)) + .ok_or(Reason::InvalidOriginHost)?; + let media_type = canonical_media_type(job.content_type); + // No address serves the optimizer's answer, so it is bound by its + // own bytes and never by the address the optimizer read. + let reference = job.source_reference.filter(|reference| { + !job.optimizer_output && safe_source_reference(reference, &self.config.domain) + }); + let mut work = Deadline::new( + services.timer(), + Duration::from_millis(self.config.max_hold_ms).saturating_sub(SKIP_WRITE_RESERVE), + ); + work.start().await; + // Asked live, as for the namespace: a cached digest could outlive a + // revocation and reuse a signed result under it. + let policy_digest = self + .request_policy(job.reader_url, services, &mut work) + .await?; + let source = MediaSource { + page_url: job.reader_url, + reference, + origin_host: Some(origin_host), + mime: job.content_type, + media_type: &media_type, + bytes: job.source, + sha256: &sha256, + link_deliverable: true, + require_embedded_manifest: job.optimizer_output, + }; + // One representation, one call at most. + let mut calls = 0_usize; + let (signed, reason) = self + .sign_media(services, &source, &policy_digest, &mut work, &mut calls) + .await?; + if media_type == SVG_MEDIA_TYPE && signed.disposition == Disposition::Embedded { + embedded::verify_svg_carrier(job.source, &signed.bytes)?; + } + let mut headers: Vec = Vec::with_capacity(4); + if signed.disposition == Disposition::External + && let Some(link) = signed.link() + { + headers.push(( + header::LINK, + HeaderValue::from_str(&link).map_err(|_| Reason::InvalidSignedResponse)?, + )); + } + for (name, value) in [ + (INSTANCE_ID_HEADER, &signed.instance_id), + (DOCUMENT_ID_HEADER, &signed.document_id), + (VERIFICATION_URL_HEADER, &signed.verification_url), + ] { + headers.push(( + name, + HeaderValue::from_str(value).map_err(|_| Reason::InvalidSignedResponse)?, + )); + } + log_media(true, reason); + Ok(DeliveredMedia { + bytes: signed.bytes, + content_type: signed.content_type, + headers, + // The policy this representation was signed under, so core keeps + // it where only that policy finds it. + namespace: self.kept_namespace(&policy_digest), + }) + } +} + +/// Marks every signed outcome of a batch whose output would exceed the +/// document budget as unsigned: none of its edits is applied. +fn revoke(outcomes: &mut [Outcome]) { + for outcome in outcomes.iter_mut() { + if outcome.is_ok() { + *outcome = Err(Reason::OutputTooLarge); + } + } +} + +/// The version of the module's signed representations, part of every +/// namespace core keeps them under. +const NAMESPACE_VERSION: &str = concat!("media-v1:", env!("CARGO_PKG_VERSION")); + +/// The module's binary media signer, which core runs on publisher and asset +/// proxy responses of the media it captures. +pub(crate) struct EncypherMedia { + signer: Arc, +} + +impl EncypherMedia { + pub(crate) fn new(signer: Arc) -> Self { + Self { signer } + } +} + +impl std::fmt::Debug for EncypherMedia { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("EncypherMedia") + .finish_non_exhaustive() + } +} + +impl MediaSigner for EncypherMedia { + fn signer_id(&self) -> &'static str { + MODULE + } + + /// Every path but an API, feed, admin or well-known path: media is + /// recognized by the origin's media type, not by a suffix. + fn handles_path(&self, path: &str) -> bool { + !is_excluded_prefix(path) + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type != MEDIA_SEGMENT_TYPE + && SIGNABLE_TYPE_PREFIXES + .iter() + .any(|prefix| media_type.starts_with(prefix) && media_type.len() > prefix.len()) + } + + fn max_source_bytes(&self) -> usize { + self.signer.max_media_bytes() + } + + /// Larger media, up to this, is signed through the prepared lane: hashed + /// where it is delivered and never uploaded. + fn max_large_source_bytes(&self) -> u64 { + self.signer.max_large_media_bytes() + } + + /// The one budget for a large source: core's whole-source hash and the + /// prepared lane together. + fn max_large_media_duration(&self) -> Duration { + self.signer.max_large_media_duration() + } + + fn sign_large<'a>( + &'a self, + services: &'a RuntimeServices, + job: &'a LargeMediaJob<'a>, + ) -> LargeMediaSigning<'a> { + Box::pin(async move { + match self.signer.sign_large(services, job).await { + Ok(signed) => { + log_media(true, Reason::Signed); + Ok(signed) + } + Err(refusal) => { + // A source with its own embedded credentials is delivered + // as it is: its provenance is kept, it is not unsigned + // by failure and not newly signed. + log::info!( + "encypher: media outcome={} lane=large reason={}", + if refusal.retains_provenance() { + "retained" + } else { + "unsigned" + }, + refusal.code() + ); + Err(MediaSigningFailure { + reason: refusal.code(), + }) + } + } + }) + } + + fn namespace<'a>(&'a self, services: &'a RuntimeServices) -> MediaNamespacing<'a> { + Box::pin(async move { + self.signer + .delivery_namespace(services) + .await + .map_err(|reason| { + log_media(false, reason); + MediaSigningFailure { + reason: reason.code(), + } + }) + }) + } + + fn sign<'a>( + &'a self, + services: &'a RuntimeServices, + job: &'a MediaJob<'a>, + ) -> MediaSigning<'a> { + Box::pin(async move { + self.signer + .sign_delivery(services, job) + .await + .map_err(|reason| { + log_media(false, reason); + MediaSigningFailure { + reason: reason.code(), + } + }) + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/provenance/encypher/src/processor/media/tests.rs b/crates/provenance/encypher/src/processor/media/tests.rs new file mode 100644 index 0000000000..fb8de7cbfa --- /dev/null +++ b/crates/provenance/encypher/src/processor/media/tests.rs @@ -0,0 +1,1188 @@ +use std::collections::HashMap; +use std::sync::Mutex; + +use edgezero_core::key_value_store::KvPage; +use futures::executor::block_on; +use serde_json::json; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use trusted_server_core::platform::{KvError, PlatformHttpClient, PlatformKvStore}; + +use super::*; +use crate::EncypherConfig; + +const TOKEN: &str = "cwk_test_worker_token_value"; +const POLICY_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const PAGE_URL: &str = "https://publisher.example.com/news/story"; +const SIGN_URL: &str = "https://api.example.com/api/v1/public/cdn/media/sign"; +const SOURCE: &[u8] = b"\x89PNG\r\n\x1a\n-source-pixels"; +const SIGNED: &[u8] = b"\x89PNG\r\n\x1a\n-source-pixels-caBX-jumb-c2pa-manifest"; +const MANIFEST: &str = "QUJD"; + +/// A runtime KV store held in memory, recording deletes. +#[derive(Default)] +struct MemoryKv { + entries: Mutex>, + deletes: Mutex>, +} + +impl MemoryKv { + fn keys_with(&self, prefix: &str) -> Vec { + self.entries + .lock() + .expect("should lock entries") + .keys() + .filter(|key| key.starts_with(prefix)) + .cloned() + .collect() + } + + fn get(&self, key: &str) -> Option { + self.entries + .lock() + .expect("should lock entries") + .get(key) + .cloned() + } + + fn set(&self, key: &str, value: Bytes) { + self.entries + .lock() + .expect("should lock entries") + .insert(key.to_owned(), value); + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + Ok(self.get(key)) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.set(key, value); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + _ttl: Duration, + ) -> Result<(), KvError> { + self.set(key, value); + Ok(()) + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.entries + .lock() + .expect("should lock entries") + .remove(key); + self.deletes + .lock() + .expect("should lock deletes") + .push(key.to_owned()); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(KvPage::default()) + } +} + +fn signer() -> Arc { + let config: EncypherConfig = serde_json::from_value(json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": TOKEN, + })) + .expect("should parse config"); + Arc::new(Signer::new(config, 5_000_000)) +} + +fn services(http: &Arc, kv: &Arc) -> RuntimeServices { + build_services_with_http_client(Arc::clone(http) as Arc) + .with_kv_store(Arc::clone(kv) as Arc) +} + +/// The signing headers of an embedded answer for `body`. +fn answer(body: &[u8], content_type: &str) -> Vec<(String, String)> { + [ + ("content-type", content_type.to_owned()), + ("x-encypher-policy-digest", POLICY_DIGEST.to_owned()), + ("x-encypher-signed-sha256", sha256_hex(body)), + ("x-encypher-manifest-instance-id", "xmp:iid:0001".to_owned()), + ("x-encypher-manifest-disposition", "embedded".to_owned()), + ("x-encypher-manifest-link-required", "false".to_owned()), + ( + "x-encypher-verification-url", + "https://verify.example.com/doc-1".to_owned(), + ), + ("x-encypher-document-id", "doc-1".to_owned()), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect() +} + +/// The signing headers of a remote-manifest answer that needs a `Link`. +fn external_answer(body: &[u8]) -> Vec<(String, String)> { + let mut headers: Vec<(String, String)> = answer(body, "image/png") + .into_iter() + .filter(|(name, _)| { + name != "x-encypher-manifest-disposition" && name != "x-encypher-manifest-link-required" + }) + .collect(); + for (name, value) in [ + ("x-encypher-manifest-disposition", "external"), + ("x-encypher-manifest-link-required", "true"), + ( + "x-encypher-manifest-url", + "https://api.example.com/manifests/doc-1", + ), + ("x-encypher-manifest-sha256", POLICY_DIGEST), + ] { + headers.push((name.to_owned(), value.to_owned())); + } + headers +} + +fn png_source(sha256: &str) -> MediaSource<'_> { + MediaSource { + page_url: PAGE_URL, + reference: None, + origin_host: None, + mime: "image/png", + media_type: "image/png", + bytes: SOURCE, + sha256, + link_deliverable: false, + require_embedded_manifest: false, + } +} + +fn sign( + signer: &Signer, + services: &RuntimeServices, + source: &MediaSource<'_>, +) -> Result<(SignedMedia, Reason), Reason> { + block_on(async { + let mut work = Deadline::Unbounded; + let mut calls = 0; + signer + .sign_media(services, source, POLICY_DIGEST, &mut work, &mut calls) + .await + }) +} + +#[test] +fn a_document_obeys_its_configured_call_limit_for_distinct_assets() { + for limit in [3, 12] { + let http = Arc::new(StubHttpClient::new()); + let images: Vec> = (0..=limit) + .map(|index| { + let mut bytes = SOURCE.to_vec(); + bytes.push(u8::try_from(index).expect("should fit")); + bytes + }) + .collect(); + for image in images.iter().take(limit) { + let mut signed = image.clone(); + signed.extend_from_slice(b"-c2pa"); + let headers = answer(&signed, "image/png"); + http.push_response_with_headers(200, signed, headers); + } + let kv = Arc::new(MemoryKv::default()); + let document: String = images + .iter() + .map(|image| { + format!( + "", + STANDARD.encode(image) + ) + }) + .collect(); + let mut config = signer().config.clone(); + config.max_media_sign_calls_per_document = limit; + let signer = Signer::new(config, 5_000_000); + let services = services(&http, &kv); + let request = page_request(); + let scan = embedded::scan_html(&document, None); + let output = block_on(async { + let mut work = Deadline::Unbounded; + signer + .sign_embedded( + &services, + &embedded_job(&request, Surface::Page), + &document, + &scan, + &mut work, + ) + .await + }) + .expect("should sign the items within the configured limit"); + + assert_eq!( + http.recorded_request_methods().len(), + limit, + "should stop calling at the configured bound" + ); + for image in images.iter().take(limit) { + let mut signed = image.clone(); + signed.extend_from_slice(b"-c2pa"); + assert!( + output.contains(&STANDARD.encode(&signed)), + "should deliver every signed distinct asset within the limit" + ); + } + let last = images.last().expect("should have a last image"); + assert!( + output.contains(&STANDARD.encode(last)), + "should leave the deferred item as written" + ); + assert_eq!( + kv.keys_with(MEDIA_SKIP_KEY_PREFIX).len(), + 0, + "a deferred item should not be negatively cached" + ); + } +} + +fn header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(candidate, _)| candidate.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) +} + +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +#[test] +fn signs_exact_bytes_once_then_serves_the_verified_cached_answer() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let signer = signer(); + let sha = sha256_hex(SOURCE); + + let (first, reason) = sign(&signer, &services, &png_source(&sha)).expect("should sign"); + + assert_eq!(reason, Reason::Signed); + assert_eq!(first.bytes.as_ref(), SIGNED, "should keep the signed bytes"); + assert_eq!(http.recorded_request_methods(), ["POST"]); + assert_eq!(http.recorded_request_uris(), [SIGN_URL]); + let headers = &http.recorded_request_headers()[0]; + let key = header(headers, "idempotency-key").expect("should send a replay key"); + assert!( + key.len() == 64 && key.bytes().all(|byte| byte.is_ascii_hexdigit()), + "should derive a hex replay key: {key}" + ); + assert!( + header(headers, "x-encypher-worker-token").is_some(), + "should authenticate with the worker token" + ); + assert!( + header(headers, "content-type") + .is_some_and(|value| value.starts_with("multipart/form-data; boundary=")), + "should send multipart" + ); + let body = &http.recorded_request_bodies()[0]; + assert!(contains(body, SOURCE), "should send the exact source bytes"); + assert!( + contains(body, b"\"reference\":null"), + "should send no reference for media without a safe address" + ); + assert!( + contains(body, b"\"origin_host\":null"), + "should send no origin host for media fetched from no host" + ); + assert!( + contains( + body, + format!("\"expected_policy_digest\":\"{POLICY_DIGEST}\"").as_bytes() + ), + "should send the held policy digest" + ); + + let (second, reason) = sign(&signer, &services, &png_source(&sha)).expect("should sign"); + assert_eq!(reason, Reason::CacheHit); + assert_eq!( + second.bytes.as_ref(), + SIGNED, + "should replay identical bytes" + ); + assert_eq!( + http.recorded_request_methods().len(), + 1, + "should make no second call" + ); +} + +#[test] +fn a_stale_policy_evicts_the_digest_and_caches_a_skip() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 409, + serde_json::to_vec(&json!({ + "success": false, + "error": {"code": "TRUSTED_SERVER_POLICY_CHANGED", "message": "stale"}, + })) + .expect("should encode"), + vec![("content-type", "application/json")], + ); + let kv = Arc::new(MemoryKv::default()); + let signer = signer(); + kv.set(&signer.policy_key, Bytes::from_static(b"{}")); + let services = services(&http, &kv); + let sha = sha256_hex(SOURCE); + + assert_eq!( + sign(&signer, &services, &png_source(&sha)).expect_err("should not sign"), + Reason::PolicyChanged + ); + assert!( + kv.get(&signer.policy_key).is_none(), + "should evict the policy" + ); + assert_eq!( + kv.keys_with(MEDIA_SKIP_KEY_PREFIX).len(), + 1, + "should cache a skip" + ); + assert_eq!( + sign(&signer, &services, &png_source(&sha)).expect_err("should not sign"), + Reason::CachedSkip + ); + assert_eq!( + http.recorded_request_methods().len(), + 1, + "should not call again" + ); +} + +#[test] +fn a_purged_result_is_asked_for_again_under_a_new_key_on_the_next_view() { + for (status, code) in [ + (410, "SIGNING_RESULT_PURGED"), + (409, "SIGNING_IDEMPOTENCY_CONFLICT"), + ] { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + status, + serde_json::to_vec(&json!({"success": false, "error": {"code": code}})) + .expect("should encode"), + ); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let signer = signer(); + let sha = sha256_hex(SOURCE); + + let first = sign(&signer, &services, &png_source(&sha)).expect_err("should not sign"); + assert!( + matches!(first, Reason::ResultPurged | Reason::IdempotencyConflict), + "{code}: should report {first:?}" + ); + assert_eq!( + http.recorded_request_methods().len(), + 1, + "{code}: should not retry" + ); + assert!( + kv.keys_with(MEDIA_SKIP_KEY_PREFIX).is_empty(), + "{code}: no skip" + ); + + let (_, reason) = sign(&signer, &services, &png_source(&sha)).expect("should sign"); + assert_eq!(reason, Reason::Signed); + let headers = http.recorded_request_headers(); + assert_ne!( + header(&headers[0], "idempotency-key"), + header(&headers[1], "idempotency-key"), + "{code}: should mint a new replay key" + ); + } +} + +#[test] +fn a_tampered_cached_answer_is_refused_and_evicted() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let signer = signer(); + let sha = sha256_hex(SOURCE); + sign(&signer, &services, &png_source(&sha)).expect("should sign"); + let key = kv.keys_with(MEDIA_KEY_PREFIX)[0].clone(); + let mut entry: serde_json::Value = + serde_json::from_slice(&kv.get(&key).expect("should hold the answer")).expect("JSON"); + entry["bytes"] = json!(STANDARD.encode(b"\x89PNG\r\n\x1a\nother-c2pa")); + kv.set(&key, Bytes::from(serde_json::to_vec(&entry).expect("JSON"))); + + assert_eq!( + sign(&signer, &services, &png_source(&sha)).expect_err("should refuse"), + Reason::CachedEntryInvalid + ); + assert!(kv.get(&key).is_none(), "should evict the tampered entry"); +} + +#[test] +fn unsigned_or_mismatched_answers_are_never_used_and_are_negatively_cached() { + let other_policy = "b".repeat(64); + let cases = [ + ( + SOURCE.to_vec(), + answer(SOURCE, "image/png"), + Reason::UnsignedResponse, + ), + ( + b"\x89PNG\r\n\x1a\nswapped-c2pa".to_vec(), + answer(SIGNED, "image/png"), + Reason::SignedDigestMismatch, + ), + ( + SIGNED.to_vec(), + answer(SIGNED, "image/jpeg"), + Reason::SignedTypeMismatch, + ), + ( + SIGNED.to_vec(), + answer(SIGNED, "image/png") + .into_iter() + .map(|(name, value)| { + if name == "x-encypher-policy-digest" { + (name, other_policy.clone()) + } else { + (name, value) + } + }) + .collect(), + Reason::PolicyDigestMismatch, + ), + ( + SIGNED.to_vec(), + external_answer(SIGNED), + Reason::ManifestUndeliverable, + ), + ( + SIGNED.to_vec(), + answer(SIGNED, "image/png") + .into_iter() + .map(|(name, value)| { + if name == "x-encypher-verification-url" { + ( + name, + "http://localhost:9000/api/v1/verify/cdnmedia_dev".to_owned(), + ) + } else { + (name, value) + } + }) + .collect(), + Reason::InvalidSignedResponse, + ), + ]; + for (body, headers, expected) in cases { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, body, headers); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let sha = sha256_hex(SOURCE); + + assert_eq!( + sign(&signer(), &services, &png_source(&sha)).expect_err("should not sign"), + expected + ); + assert!( + kv.keys_with(MEDIA_KEY_PREFIX).is_empty(), + "{expected:?}: should cache no answer" + ); + assert_eq!( + kv.keys_with(MEDIA_SKIP_KEY_PREFIX).len(), + 1, + "{expected:?}: should cache the failure" + ); + } +} + +fn embedded_job<'a>(request: &'a PageRequest, surface: Surface) -> EmbeddedJob<'a> { + EmbeddedJob { + request, + surface, + policy_digest: POLICY_DIGEST, + } +} + +fn page_request() -> PageRequest { + PageRequest { + method: Method::GET, + page_url: PAGE_URL.to_owned(), + removed_prefix: String::new(), + response_status: 200, + origin_response_is_private: false, + } +} + +fn sign_document(services: &RuntimeServices, document: &str, surface: Surface) -> Option { + sign_document_with_policies(services, document, surface, None) +} + +fn sign_document_with_policies( + services: &RuntimeServices, + document: &str, + surface: Surface, + policies: Option<&[String]>, +) -> Option { + let request = page_request(); + let scan = match surface { + Surface::Page => embedded::scan_html(document, policies), + Surface::Stylesheet => embedded::scan_stylesheet(document), + }; + block_on(async { + let mut work = Deadline::Unbounded; + signer() + .sign_embedded( + services, + &embedded_job(&request, surface), + document, + &scan, + &mut work, + ) + .await + }) +} + +#[test] +fn embedded_data_uris_are_replaced_by_their_signed_bytes_once_per_source() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let source = format!("data:image/png;base64,{}", STANDARD.encode(SOURCE)); + let signed = format!("data:image/png;base64,{}", STANDARD.encode(SIGNED)); + let document = format!( + "

x

a" + ); + + let output = sign_document(&services(&http, &kv), &document, Surface::Page) + .expect("should sign the page's media"); + + assert_eq!( + output, + format!( + "

x

a" + ), + "should replace each URI and change nothing else" + ); + assert_eq!( + http.recorded_request_methods(), + ["POST"], + "should sign one source once" + ); +} + +#[test] +fn a_failed_embedded_signature_leaves_the_uri_as_written() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 422, + b"{\"error\":{\"code\":\"UNSUPPORTED_MEDIA_TYPE\"}}".to_vec(), + ); + let kv = Arc::new(MemoryKv::default()); + let document = format!( + "", + STANDARD.encode(SOURCE) + ); + + assert_eq!( + sign_document(&services(&http, &kv), &document, Surface::Page), + None, + "should leave the page as written" + ); +} + +#[test] +fn stylesheet_media_is_signed_in_place() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let document = format!( + ".hero{{background:url('data:image/png;base64,{}') no-repeat}}", + STANDARD.encode(SOURCE) + ); + + let output = sign_document(&services(&http, &kv), &document, Surface::Stylesheet) + .expect("should sign the stylesheet's media"); + + assert_eq!( + output, + format!( + ".hero{{background:url('data:image/png;base64,{}') no-repeat}}", + STANDARD.encode(SIGNED) + ) + ); +} + +#[test] +fn inline_svg_is_signed_as_its_exact_bytes_and_kept_only_with_the_writer_s_carrier_alone() { + let root = ""; + let body = ""; + let svg = format!("{root}{body}"); + let document = format!("

x

{svg}

y

"); + let signed_svg = format!("{root}{MANIFEST}{body}"); + let restyled = signed_svg.replace("currentColor", "#000"); + for (answer_svg, expected) in [ + ( + signed_svg.clone(), + Some(format!("

x

{signed_svg}

y

")), + ), + (restyled, None), + ] { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 200, + answer_svg.clone().into_bytes(), + answer(answer_svg.as_bytes(), "image/svg+xml"), + ); + let kv = Arc::new(MemoryKv::default()); + + let output = sign_document(&services(&http, &kv), &document, Surface::Page); + + assert_eq!( + output, expected, + "should keep only a pure manifest insertion" + ); + assert!( + contains(&http.recorded_request_bodies()[0], svg.as_bytes()), + "should send the element's exact bytes" + ); + } +} + +#[test] +fn inline_svg_the_browser_reads_differently_is_signed_as_its_parsed_tree() { + // The browser adds no namespace and keeps the first `viewBox`; what is + // hashed, sent, compared and placed is that tree as standalone XML. + let original = ""; + let serialized = crate::svg_equivalence::signable_svg(original, 1024 * 1024) + .expect("should have an equivalent standalone document") + .into_owned(); + assert_ne!(serialized, original, "should not sign the bytes as written"); + let document = roxmltree::Document::parse(&serialized).expect("should be standalone XML"); + assert_eq!( + document.root_element().attribute("viewBox"), + Some("0 0 2 2"), + "should keep the browser's first viewBox" + ); + // The writer's carrier goes right after the root start tag, whose + // attribute values escape any `>`. + let (root, body) = + serialized.split_at(serialized.find('>').expect("should close the root") + 1); + let signed_svg = format!("{root}{MANIFEST}{body}"); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 200, + signed_svg.clone().into_bytes(), + answer(signed_svg.as_bytes(), "image/svg+xml"), + ); + let kv = Arc::new(MemoryKv::default()); + + let output = sign_document( + &services(&http, &kv), + &format!("

x

{original}

y

"), + Surface::Page, + ); + + assert_eq!( + output, + Some(format!("

x

{signed_svg}

y

")), + "should place the signed serialization where the element was" + ); + let sent = &http.recorded_request_bodies()[0]; + assert!( + contains(sent, serialized.as_bytes()), + "should send the serialized tree" + ); + assert!( + !contains(sent, original.as_bytes()), + "should not send bytes the browser reads differently" + ); +} + +#[test] +fn unsigned_self_closing_svgs_never_add_whole_asset_signing_calls() { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + assert_eq!( + sign_document(&services(&http, &kv), "", Surface::Page), + None + ); + assert!(http.recorded_request_methods().is_empty()); + + let source = format!("data:image/png;base64,{}", STANDARD.encode(SOURCE)); + let signed = format!("data:image/png;base64,{}", STANDARD.encode(SIGNED)); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + assert_eq!( + sign_document( + &services(&http, &kv), + &format!(""), + Surface::Page, + ), + Some(format!("")) + ); + assert_eq!(http.recorded_request_methods(), ["POST"]); +} + +#[test] +fn original_svg_provenance_makes_no_child_or_whole_asset_signing_calls() { + let source = format!("data:image/png;base64,{}", STANDARD.encode(SOURCE)); + for provenance in [ + MANIFEST, + "", + "https://example.com/m", + "", + "", + ] { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + let document = format!( + "\ + {provenance}" + ); + assert_eq!( + sign_document_with_policies(&services(&http, &kv), &document, Surface::Page, Some(&[]),), + None, + "must preserve the entire original range for {provenance}" + ); + assert!(http.recorded_request_methods().is_empty(), "{provenance}"); + } +} + +#[test] +fn signing_an_outside_image_cannot_change_original_svg_provenance_ranges() { + let source = format!("data:image/png;base64,{}", STANDARD.encode(SOURCE)); + let signed = format!("data:image/png;base64,{}", STANDARD.encode(SIGNED)); + let svg = format!( + "{MANIFEST}\ + \ + " + ); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let document = format!("{svg}"); + assert_eq!( + sign_document_with_policies(&services(&http, &kv), &document, Surface::Page, Some(&[]),), + Some(format!("{svg}")), + "the rescan after outside edits must still preserve every SVG byte" + ); + assert_eq!(http.recorded_request_methods(), ["POST"]); +} + +#[test] +fn an_unsigned_svg_still_signs_its_attribute_css_and_then_the_whole_asset() { + let source = format!("data:image/png;base64,{}", STANDARD.encode(SOURCE)); + let signed = format!("data:image/png;base64,{}", STANDARD.encode(SIGNED)); + let root = ""; + let body = format!( + "Signed with C2PA\ + " + ); + let source_svg = format!("{root}{body}"); + let child_signed_svg = source_svg.replace(&source, &signed); + let signed_svg = format!("{root}{MANIFEST}{}", body.replace(&source, &signed)); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + http.push_response_with_headers( + 200, + signed_svg.as_bytes().to_vec(), + answer(signed_svg.as_bytes(), "image/svg+xml"), + ); + let kv = Arc::new(MemoryKv::default()); + assert_eq!( + sign_document_with_policies(&services(&http, &kv), &source_svg, Surface::Page, Some(&[]),), + Some(signed_svg) + ); + assert_eq!(http.recorded_request_methods(), ["POST", "POST"]); + let requests = http.recorded_request_bodies(); + assert!(contains(&requests[0], SOURCE)); + assert!(contains(&requests[1], child_signed_svg.as_bytes())); + assert!(!contains(&requests[1], source_svg.as_bytes())); +} + +#[test] +fn inline_svg_that_breaks_out_is_reported_without_a_call() { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + + let output = sign_document( + &services(&http, &kv), + "

x

", + Surface::Page, + ); + + assert_eq!(output, None); + assert!( + http.recorded_request_methods().is_empty(), + "should not call" + ); +} + +fn job<'a>( + reader_url: &'a str, + source: &'a Bytes, + sha256: &'a str, + reference: Option<&'a str>, + origin_host: Option<&'a str>, +) -> MediaJob<'a> { + MediaJob { + reader_url, + content_type: "image/png", + source, + source_sha256: sha256, + source_reference: reference, + source_origin_host: origin_host, + optimizer_output: false, + } +} + +fn deliver( + services: &RuntimeServices, + job: &MediaJob<'_>, +) -> Result { + let media = EncypherMedia::new(signer()); + block_on(media.sign(services, job)) +} + +const ORIGIN: Option<&str> = Some("origin.example.com"); + +#[test] +fn delivered_media_outside_the_claim_with_other_bytes_or_no_origin_host_is_refused() { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let source = Bytes::from_static(SOURCE); + let sha = sha256_hex(SOURCE); + let reader = "https://publisher.example.com/a.png"; + + let outside = deliver( + &services, + &job("https://cdn.example.org/a.png", &source, &sha, None, ORIGIN), + ) + .expect_err("should refuse another host"); + assert_eq!(outside.reason, "outside_claim"); + let other = "0".repeat(64); + let mismatch = deliver(&services, &job(reader, &source, &other, None, ORIGIN)) + .expect_err("should refuse other bytes"); + assert_eq!(mismatch.reason, "source_digest_mismatch"); + for host in [ + None, + Some(""), + Some("origin.example.com:8443"), + Some("user@origin.example.com"), + Some("origin.example.com/images"), + Some("Origin.example.com"), + ] { + let refused = deliver(&services, &job(reader, &source, &sha, None, host)) + .expect_err("should refuse a missing or malformed origin host"); + assert_eq!(refused.reason, "invalid_origin_host", "{host:?}"); + } + assert!( + http.recorded_request_methods().is_empty(), + "should not call" + ); +} + +#[test] +fn delivered_media_carries_a_link_for_a_remote_manifest_and_only_a_safe_reference() { + for (reference, origin_host, expected_reference, expected_origin) in [ + ( + Some("https://publisher.example.com/images/a.png"), + "publisher.example.com", + "\"reference\":\"https://publisher.example.com/images/a.png\"", + "\"origin_host\":\"publisher.example.com\"", + ), + ( + Some("https://origin.example.com/images/a.png"), + "origin.example.com", + "\"reference\":null", + "\"origin_host\":\"origin.example.com\"", + ), + // A fetched address with a query is no reference, but its host is + // still the origin's. + ( + None, + "origin.example.com", + "\"reference\":null", + "\"origin_host\":\"origin.example.com\"", + ), + ] { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&json!({"policy_digest": POLICY_DIGEST})).expect("JSON"), + ); + http.push_response_with_headers(200, SOURCE.to_vec(), external_answer(SOURCE)); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let source = Bytes::from_static(SOURCE); + let sha = sha256_hex(SOURCE); + + let delivered = deliver( + &services, + &job( + "https://publisher.example.com/images/a.png", + &source, + &sha, + reference, + Some(origin_host), + ), + ) + .expect("should sign"); + + assert_eq!( + delivered.bytes.as_ref(), + SOURCE, + "should deliver the answer's bytes" + ); + assert_eq!(delivered.content_type, "image/png"); + assert_eq!( + delivered.namespace, + format!( + "{NAMESPACE_VERSION}:{POLICY_DIGEST}:{}", + signer().cache_context + ), + "should name the policy it was signed under for core's kept copy" + ); + let named = |name: &str| { + delivered + .headers + .iter() + .find(|(header, _)| header.as_str() == name) + .map(|(_, value)| value.to_str().expect("text").to_owned()) + }; + assert_eq!( + named("link").as_deref(), + Some("; rel=\"c2pa-manifest\""), + "should name the remote manifest on every response" + ); + assert_eq!(named("x-encypher-document-id").as_deref(), Some("doc-1")); + let body = &http.recorded_request_bodies()[1]; + assert!( + contains(body, expected_reference.as_bytes()), + "should send {expected_reference}" + ); + assert!( + contains(body, expected_origin.as_bytes()), + "should send {expected_origin}" + ); + assert!( + contains(body, b"\"http_link_header\":true"), + "should say a Link can be delivered" + ); + } +} + +#[test] +fn an_optimizer_answer_is_sent_requiring_its_manifest_and_bound_by_its_bytes() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&json!({"policy_digest": POLICY_DIGEST})).expect("JSON"), + ); + http.push_response_with_headers(200, SOURCE.to_vec(), external_answer(SOURCE)); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let source = Bytes::from_static(SOURCE); + let sha = sha256_hex(SOURCE); + let optimized = MediaJob { + optimizer_output: true, + ..job( + "https://publisher.example.com/images/a.png", + &source, + &sha, + Some("https://publisher.example.com/images/a.png"), + Some("publisher.example.com"), + ) + }; + + deliver(&services, &optimized).expect("should sign"); + + let body = &http.recorded_request_bodies()[1]; + assert!( + contains(body, b"\"require_embedded_manifest\":true"), + "should require the optimizer's embedded manifest" + ); + assert!( + contains(body, b"\"reference\":null"), + "should not name the address the optimizer read as the source" + ); + assert!( + contains(body, b"\"operations\":[]"), + "should report no operation of the edge's own" + ); +} + +#[test] +fn a_result_is_never_reused_for_the_same_bytes_from_another_origin() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let signer = signer(); + let sha = sha256_hex(SOURCE); + + for host in ["origin-a.example.com", "origin-b.example.com"] { + let source = MediaSource { + origin_host: Some(host), + ..png_source(&sha) + }; + let (_, reason) = sign(&signer, &services, &source).expect("should sign"); + assert_eq!(reason, Reason::Signed, "{host}: should ask the endpoint"); + } + assert_eq!( + http.recorded_request_methods().len(), + 2, + "should bind each result to its origin" + ); + let headers = http.recorded_request_headers(); + assert_ne!( + header(&headers[0], "idempotency-key"), + header(&headers[1], "idempotency-key"), + "should replay neither under the other's key" + ); +} + +#[test] +fn media_is_recognized_by_its_type_on_every_path_but_excluded_prefixes() { + let media = EncypherMedia::new(signer()); + for path in [ + "/images/a.png", + "/video/clip.MOV", + "/audio/a.mp3", + "/img/123", + "/media", + ] { + assert!(media.handles_path(path), "should consider {path}"); + } + for path in [ + "/api/v1/a.png", + "/wp-admin/x.jpg", + "/.well-known/a.png", + "/feed/", + ] { + assert!(!media.handles_path(path), "should exclude {path}"); + } + for media_type in [ + "image/webp", + "image/svg+xml", + "video/mp4", + "video/webm", + "audio/mpeg", + "font/woff2", + ] { + assert!( + media.handles_media_type(media_type), + "should sign {media_type}" + ); + } + for media_type in [ + "image/", + "text/css", + "text/html", + "application/pdf", + "video/iso.segment", + ] { + assert!( + !media.handles_media_type(media_type), + "should not sign {media_type}" + ); + } + assert_eq!(media.max_source_bytes(), 5_000_000, "should bound captures"); +} + +#[test] +fn the_kept_copy_namespace_fails_closed_on_revocation_however_warm_the_caches() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&json!({"policy_digest": POLICY_DIGEST})).expect("JSON"), + ); + http.push_response(403, b"{}".to_vec()); + let kv = Arc::new(MemoryKv::default()); + let runtime = services(&http, &kv); + let media = EncypherMedia::new(signer()); + // The article path's policy cache holds a fresh digest throughout. + let signer = signer(); + kv.set( + &signer.policy_key, + Bytes::from( + serde_json::to_vec(&json!({ + "kind": "digest", + "policy_digest": POLICY_DIGEST, + "expires_at": unix_now() + 300, + })) + .expect("JSON"), + ), + ); + + let namespace = block_on(media.namespace(&runtime)).expect("should answer a namespace"); + assert!( + namespace.contains(POLICY_DIGEST), + "should bind the policy digest: {namespace}" + ); + assert!( + !namespace.contains(TOKEN), + "should never carry the credential" + ); + let failure = block_on(media.namespace(&runtime)) + .expect_err("should refuse a revoked credential with the cached digest still fresh"); + assert_eq!(failure.reason, "forbidden"); +} + +#[test] +fn a_warm_signed_result_is_not_delivered_under_a_revoked_credential() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&json!({"policy_digest": POLICY_DIGEST})).expect("JSON"), + ); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + http.push_response(403, b"{}".to_vec()); + let kv = Arc::new(MemoryKv::default()); + let runtime = services(&http, &kv); + let source = Bytes::from_static(SOURCE); + let sha = sha256_hex(SOURCE); + let job = job( + "https://publisher.example.com/images/a.png", + &source, + &sha, + None, + ORIGIN, + ); + deliver(&runtime, &job).expect("should sign and cache the result"); + assert!( + !kv.keys_with(MEDIA_KEY_PREFIX).is_empty(), + "should hold the signed result" + ); + + let refused = deliver(&runtime, &job).expect_err("should refuse the revoked credential"); + assert_eq!(refused.reason, "forbidden"); + assert_eq!( + http.recorded_request_methods(), + ["GET", "POST", "GET"], + "should neither sign again nor deliver the cached result" + ); +} diff --git a/crates/provenance/encypher/src/processor/prepared.rs b/crates/provenance/encypher/src/processor/prepared.rs new file mode 100644 index 0000000000..8aa58ad3ad --- /dev/null +++ b/crates/provenance/encypher/src/processor/prepared.rs @@ -0,0 +1,1087 @@ +//! Large media signed where it is delivered, through the prepared lane. +//! +//! A source too large to capture is never uploaded and never held whole. +//! Core pins one version of it by its strong `ETag` and hashes it in bounded +//! chunks. This lane then asks the Encypher prepared-signing endpoints for a +//! remote manifest over those exact bytes: +//! +//! 1. The policy, then the lane's capabilities for the asset. +//! 2. The signed probe release the service advertises, verified against the +//! pinned release key before it runs. +//! 3. The probe's disclosure of the source's container structure, read from +//! the pinned version. A source that already carries an embedded C2PA +//! manifest stops here: a remote manifest cannot stand beside it, as an +//! embedded manifest wins over a `Link`, so the source is delivered as it +//! is, with its own credentials, reported as existing provenance kept and +//! never as newly signed or verified. +//! 4. `prepare`, then the probe's check that the server's plan covers the +//! whole source unchanged, then a last check that the version is still +//! current, then `finalize`. +//! 5. The returned store is checked to bind the whole-source digest. The +//! reader gets the origin's bytes with a `Link` to that store. +//! +//! Every failure is a refusal with a code. Core then delivers the origin's +//! bytes unsigned; nothing is published for a source the lane did not +//! verify. + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use http::{HeaderValue, Method, Request, StatusCode, header}; +use serde::Serialize; +use serde_json::{Value, json}; +use std::time::Duration; +use trusted_server_core::media_signing::{ + LargeMediaJob, LargeSignedMedia, MediaSourceError, VersionedMediaSource, +}; +use trusted_server_core::middleware::ResponseHeader; +use trusted_server_core::platform::RuntimeServices; + +use super::{ + Deadline, Reason, SKIP_WRITE_RESERVE, Signer, WORKER_TOKEN_HEADER, is_excluded_prefix, +}; +use crate::api::valid_policy_digest; +use crate::media_api::{ + self, IDEMPOTENCY_KEY_HEADER, canonical_media_type, safe_source_reference, sha256_hex, + valid_origin_host, +}; +use crate::prepared_probe::{ + ProbeCallError, ProbeDescriptor, ProbeLimits, ProbeModule, ProbeSource, ReleaseAuthority, +}; + +const CAPABILITIES_PATH: &str = "/api/v1/public/cdn/media/hash/capabilities"; +const PREPARE_PATH: &str = "/api/v1/public/cdn/media/hash/prepare"; +const FINALIZE_PATH: &str = "/api/v1/public/cdn/media/hash/finalize"; +/// Signed probe releases, served without credentials and never changed. +const PROBE_PATH: &str = "/api/v1/"; +const PROTOCOL: &str = "encypher-prepared-signing/2"; +/// The cached artifact of a probe release, by its SHA-256. +const PROBE_KEY_PREFIX: &str = "encypher:probe:v1"; +/// The largest probe artifact accepted. +const MAX_PROBE_BYTES: usize = 4 * 1024 * 1024; +/// A capabilities answer is a few descriptors and limits. +const MAX_CAPABILITIES_BYTES: usize = 64 * 1024; +/// A prepare answer is ids and the planner's geometry. +const MAX_PREPARE_BYTES: usize = 1024 * 1024; +/// A finalize answer carries the manifest store, base64 encoded, besides +/// ids and URLs. +const FINALIZE_OVERHEAD_BYTES: usize = 64 * 1024; +/// The largest finalized manifest store the lane takes, whatever the +/// service allows: the answer, its decoded store and the probe's copy of it +/// must fit an edge instance. An external manifest of an unsigned source, +/// with no ingredient chain or identity assertion, is far smaller. +const MAX_FINALIZED_MANIFEST_BYTES: usize = 2 * 1024 * 1024; +/// The refusal for a finalized manifest larger than the lane takes. +const MANIFEST_TOO_LARGE: &str = "manifest_too_large"; +/// The outcome of a source that already carries an embedded C2PA manifest: +/// delivered as it is, its own provenance kept, nothing newly signed. +const EMBEDDED_PROVENANCE: &str = "source_has_embedded_provenance"; +/// The outcome of a source whose XMP already names a C2PA manifest: delivered +/// as it is, its own provenance kept, nothing newly signed. +const REFERENCED_PROVENANCE: &str = "source_has_provenance_reference"; + +const INSTANCE_ID_HEADER: &str = "x-encypher-manifest-instance-id"; +const DOCUMENT_ID_HEADER: &str = "x-encypher-document-id"; +const VERIFICATION_URL_HEADER: &str = "x-encypher-verification-url"; + +/// Why the prepared lane did not sign a large source. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum LargeRefusal { + /// A refusal the module shares with its other lanes. + Module(Reason), + /// The pinned source could not be read as the version it was pinned at. + Source(MediaSourceError), + /// A refusal particular to this lane. + Lane(&'static str), +} + +impl LargeRefusal { + pub(super) fn code(self) -> &'static str { + match self { + Self::Module(reason) => reason.code(), + Self::Source(error) => error.code(), + Self::Lane(code) => code, + } + } + + /// Whether the source was delivered with its own embedded credentials, + /// its provenance kept rather than lost to a failure. + pub(super) fn retains_provenance(self) -> bool { + self == Self::Lane(EMBEDDED_PROVENANCE) || self == Self::Lane(REFERENCED_PROVENANCE) + } +} + +impl From for LargeRefusal { + fn from(reason: Reason) -> Self { + Self::Module(reason) + } +} + +impl From for LargeRefusal { + fn from(error: MediaSourceError) -> Self { + Self::Source(error) + } +} + +/// The pinned version, as the probe reads it. +struct PinnedSource<'a>(&'a VersionedMediaSource<'a>); + +impl ProbeSource for PinnedSource<'_> { + type Error = MediaSourceError; + + fn size(&self) -> u64 { + self.0.size() + } + + async fn read_exact_at( + &self, + offset: u64, + destination: &mut [u8], + ) -> Result<(), MediaSourceError> { + self.0.read_exact_at(offset, destination).await + } +} + +/// The lane's capabilities for one asset, as the service answers them. +struct Capabilities { + policy_digest: String, + formats: Vec, + total_size_bytes: u64, + manifest_decoded_bytes: usize, + probes: Vec, +} + +impl Capabilities { + /// The largest finalized store the lane takes: the service's maximum, + /// never more than the edge's own. + fn manifest_limit(&self) -> usize { + self.manifest_decoded_bytes + .min(MAX_FINALIZED_MANIFEST_BYTES) + } + + fn parse(body: &[u8]) -> Option { + let value: Value = serde_json::from_slice(body).ok()?; + if value.get("protocol")?.as_str()? != PROTOCOL || !value.get("enabled")?.as_bool()? { + return None; + } + let maximums = value.get("maximums")?; + Some(Self { + policy_digest: value.get("policy_digest")?.as_str()?.to_owned(), + formats: value + .get("formats")? + .as_array()? + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + total_size_bytes: maximums.get("total_size_bytes")?.as_u64()?, + manifest_decoded_bytes: usize::try_from( + maximums.get("manifest_decoded_bytes")?.as_u64()?, + ) + .ok()?, + probes: ProbeDescriptor::parse_list(value.get("probe")), + }) + } +} + +/// What the lane signs one source with, once the gates passed. +#[derive(Clone, Copy)] +struct Lane<'a> { + job: &'a LargeMediaJob<'a>, + reference: &'a str, + mime: &'a str, + policy_digest: &'a str, + /// The release key probes are held to: production, except in tests. + authority: &'a ReleaseAuthority, +} + +/// The prepare request, serialized from borrowed values so the disclosure +/// is not copied. +#[derive(Serialize)] +struct PrepareBody<'a> { + page_url: &'a str, + expected_policy_digest: &'a str, + protocol: &'a str, + probe: ProbeSelection<'a>, + source: PreparedSource<'a>, + structure: Vec>, +} + +#[derive(Serialize)] +struct ProbeSelection<'a> { + sha256: &'a str, + planner_id: &'a str, +} + +#[derive(Serialize)] +struct PreparedSource<'a> { + reference: &'a str, + origin_host: &'a str, + mime: &'a str, + sha256: &'a str, + byte_length: u64, +} + +#[derive(Serialize)] +struct StructureChunk<'a> { + offset: u64, + b64: &'a str, +} + +/// The verified finalize answer: what the delivered headers name. +struct Finalized { + manifest_url: String, + instance_id: String, + document_id: String, + verification_url: String, +} + +/// Whether `value` is an identifier safe in a response header. +fn valid_identifier(value: &str) -> bool { + !value.is_empty() && value.len() <= 256 && value.bytes().all(|byte| byte.is_ascii_graphic()) +} + +/// Whether `value` is a canonical absolute `https` URL safe inside a `Link`. +fn valid_https_url(value: &str) -> bool { + value.len() <= 2048 + && value + .bytes() + .all(|byte| byte.is_ascii_graphic() && !matches!(byte, b'<' | b'>' | b'"')) + && url::Url::parse(value).is_ok_and(|parsed| { + parsed.scheme() == "https" && parsed.host_str().is_some() && parsed.as_str() == value + }) +} + +/// The reason a probe call failed, logged with the probe's own code. +fn probe_refusal>(op: &str, error: ProbeCallError) -> LargeRefusal { + log::info!("encypher: large media probe op={op} code={}", error.code()); + match error { + ProbeCallError::Source(error) => error.into(), + ProbeCallError::Refused(refusal) if refusal.integrity => { + LargeRefusal::Lane("source_provenance_invalid") + } + ProbeCallError::Refused(_) => LargeRefusal::Lane("probe_refused"), + ProbeCallError::Failed(_) => LargeRefusal::Lane("probe_failed"), + } +} + +/// Whether `items` is the one program of an unchanged external plan: the +/// whole source, once, with nothing excluded or inserted. +fn covers_whole_source(items: &[Value], size: u64) -> bool { + matches!(items, [item] if item.get("range").and_then(Value::as_array).is_some_and(|range| { + range.len() == 2 + && range.first().and_then(Value::as_u64) == Some(0) + && range.get(1).and_then(Value::as_u64) == Some(size) + }) && item.as_object().is_some_and(|object| object.len() == 1)) +} + +/// The reason for an error answer of the prepared endpoints, by status and +/// failure code. +fn answer_refusal(status: StatusCode, code: Option<&str>) -> LargeRefusal { + match (status.as_u16(), code) { + (409, Some("TRUSTED_SERVER_POLICY_CHANGED")) => Reason::PolicyChanged.into(), + (409, Some("PROBE_VERSION_UNSUPPORTED" | "PLAN_STALE")) => { + LargeRefusal::Lane("probe_release_stale") + } + (409, Some("SOURCE_DIGEST_MISMATCH")) => LargeRefusal::Lane("source_digest_mismatch"), + (422, Some("SOURCE_PROVENANCE_INVALID")) => LargeRefusal::Lane("source_provenance_invalid"), + // A store the probe's disclosure did not report: kept, not signed. + (422, Some("SOURCE_MANIFEST_PRESENT")) => LargeRefusal::Lane(EMBEDDED_PROVENANCE), + (422, Some("HASH_MODE_UNSUPPORTED")) => LargeRefusal::Lane("format_unsupported"), + (403 | 404, _) => Reason::Forbidden.into(), + (409, _) => LargeRefusal::Lane("prepared_conflict"), + (413, _) => Reason::PayloadTooLarge.into(), + (422, _) => Reason::Unprocessable.into(), + (429, _) => Reason::RateLimited.into(), + (500..=599, _) => Reason::ServerError.into(), + _ => Reason::UnexpectedStatus.into(), + } +} + +impl Signer { + /// Largest source the prepared lane signs; larger media is delivered + /// unsigned. + pub(super) fn max_large_media_bytes(&self) -> u64 { + self.config.max_large_media_bytes + } + + /// The whole time core and this lane may spend on one large source, or + /// none when the lane is off. + pub(super) fn max_large_media_duration(&self) -> Duration { + if self.config.max_large_media_bytes == 0 { + Duration::ZERO + } else { + Duration::from_millis(self.config.max_large_media_ms) + } + } + + /// Sign one large source core pinned and hashed, returning the headers + /// every response of it carries and the namespace it was signed under. + /// Probe releases are always held to the pinned production key. + pub(super) async fn sign_large( + &self, + services: &RuntimeServices, + job: &LargeMediaJob<'_>, + ) -> Result { + self.sign_large_with(services, job, &ReleaseAuthority::production()) + .await + } + + /// [`Self::sign_large`] with the release `authority` given, which only + /// tests set to anything but production. + async fn sign_large_with( + &self, + services: &RuntimeServices, + job: &LargeMediaJob<'_>, + authority: &ReleaseAuthority, + ) -> Result { + let source = job.source; + let size = source.size(); + let reader_url = url::Url::parse(job.reader_url).map_err(|_| Reason::OutsideClaim)?; + let reader_host = reader_url.host_str().ok_or(Reason::OutsideClaim)?; + if !matches!(reader_url.scheme(), "https" | "http") + || !self.claims_host(reader_host) + || is_excluded_prefix(reader_url.path()) + { + return Err(Reason::OutsideClaim.into()); + } + if size == 0 || size > self.max_large_media_bytes() { + return Err(Reason::MediaTooLarge.into()); + } + if !valid_origin_host(job.source_origin_host) { + return Err(Reason::InvalidOriginHost.into()); + } + if !valid_policy_digest(job.source_sha256) { + return Err(Reason::SourceDigestMismatch.into()); + } + // The fetched address when it names the origin's bytes; otherwise + // the reader's own address, which names the same bytes here because + // they are delivered unchanged. A source with neither is refused. + let reference = job + .source_reference + .filter(|reference| safe_source_reference(reference, job.source_origin_host)) + .or_else(|| { + job.reader_reference.filter(|reference| { + *reference == job.reader_url && safe_source_reference(reference, reader_host) + }) + }) + .ok_or(LargeRefusal::Lane("source_reference_required"))?; + let mime = canonical_media_type(job.content_type); + + // What is left of the lane's budget after core hashed the source, + // less the reserve a policy eviction may still use. + let mut work = Deadline::new( + services.timer(), + job.remaining.saturating_sub(SKIP_WRITE_RESERVE), + ); + work.start().await; + // Asked live: the namespace core keeps the result under must be the + // policy the service holds now, never a cached one a revocation + // outlived. + let policy_digest = self + .request_policy(job.reader_url, services, &mut work) + .await?; + let lane = Lane { + job, + reference, + mime: &mime, + policy_digest: &policy_digest, + authority, + }; + let result = self.run_prepared(services, &lane, &mut work).await; + if let Err(LargeRefusal::Module(Reason::PolicyChanged | Reason::PolicyDigestMismatch)) = + &result + { + self.evict_policy(services.kv_store(), &mut Self::skip_deadline(services)) + .await; + } + let finalized = result?; + let link = format!("<{}>; rel=\"c2pa-manifest\"", finalized.manifest_url); + let mut headers: Vec = Vec::with_capacity(4); + for (name, value) in [ + (header::LINK, link.as_str()), + ( + http::HeaderName::from_static(INSTANCE_ID_HEADER), + finalized.instance_id.as_str(), + ), + ( + http::HeaderName::from_static(DOCUMENT_ID_HEADER), + finalized.document_id.as_str(), + ), + ( + http::HeaderName::from_static(VERIFICATION_URL_HEADER), + finalized.verification_url.as_str(), + ), + ] { + headers.push(( + name, + HeaderValue::from_str(value).map_err(|_| Reason::InvalidSignedResponse)?, + )); + } + Ok(LargeSignedMedia { + headers, + namespace: self.kept_namespace(&policy_digest), + }) + } + + /// The lane's steps after the gates: capabilities, probe, disclosure, + /// prepare, plan check, version check, finalize, store check. + async fn run_prepared( + &self, + services: &RuntimeServices, + lane: &Lane<'_>, + work: &mut Deadline<'_>, + ) -> Result { + let Lane { + job, + reference, + mime, + policy_digest, + authority, + } = *lane; + let source = job.source; + let size = source.size(); + let capabilities = self.capabilities(services, job.reader_url, work).await?; + if capabilities.policy_digest != policy_digest { + return Err(Reason::PolicyChanged.into()); + } + if !capabilities.formats.iter().any(|format| format == mime) { + return Err(LargeRefusal::Lane("format_unsupported")); + } + if size > capabilities.total_size_bytes { + return Err(Reason::MediaTooLarge.into()); + } + let probe = self + .load_probe(services, authority, &capabilities.probes, work) + .await?; + let pinned = PinnedSource(source); + + let mut extracted = match work + .race(probe.call( + "extract", + &json!({"mime": mime, "total_size": size, "disposition": "external"}), + &pinned, + )) + .await + { + None => return Err(Reason::Timeout.into()), + Some(result) => result.map_err(|error| probe_refusal("extract", error))?, + }; + // An embedded manifest wins over a `Link`, so a remote one would be + // ignored beside it, and a source whose XMP already names a manifest + // has provenance of its own: either way the source keeps it. A + // release that does not say whether the XMP names one cannot be + // trusted to have looked. + if extracted.get("credentials").is_some() { + return Err(LargeRefusal::Lane(EMBEDDED_PROVENANCE)); + } + match extracted + .get("has_provenance_reference") + .and_then(Value::as_bool) + { + Some(true) => return Err(LargeRefusal::Lane(REFERENCED_PROVENANCE)), + Some(false) => {} + None => return Err(LargeRefusal::Lane("probe_result_invalid")), + } + // The disclosure is taken out of the result, not copied: it is sent + // at prepare and handed back to the probe at plan verification. + let disclosure = extracted + .get_mut("chunks") + .map(Value::take) + .ok_or(LargeRefusal::Lane("probe_result_invalid"))?; + drop(extracted); + let structure = disclosure + .as_array() + .ok_or(LargeRefusal::Lane("probe_result_invalid"))? + .iter() + .map(|chunk| { + Some(StructureChunk { + offset: chunk.get("offset")?.as_u64()?, + b64: chunk.get("data")?.as_str()?, + }) + }) + .collect::>>() + .ok_or(LargeRefusal::Lane("probe_result_invalid"))?; + + let idempotency_key = sha256_hex( + format!( + "prepared\n{}\n{policy_digest}\n{}\n{reference}\n{}\n{mime}\n{}\n{size}\n{}", + self.cache_context, + job.reader_url, + job.source_origin_host, + job.source_sha256, + source.version().etag(), + ) + .as_bytes(), + ); + let descriptor = probe.descriptor(); + let prepare = PrepareBody { + page_url: job.reader_url, + expected_policy_digest: policy_digest, + protocol: PROTOCOL, + probe: ProbeSelection { + sha256: &descriptor.sha256, + planner_id: &descriptor.planner_id, + }, + source: PreparedSource { + reference, + origin_host: job.source_origin_host, + mime, + sha256: job.source_sha256, + byte_length: size, + }, + structure, + }; + let mut prepared = self + .post( + services, + PREPARE_PATH, + &prepare, + &idempotency_key, + MAX_PREPARE_BYTES, + work, + ) + .await?; + drop(prepare); + let plan_id = prepared + .get("plan_id") + .and_then(Value::as_str) + .filter(|id| id.len() == 36) + .ok_or(LargeRefusal::Lane("prepare_answer_invalid"))? + .to_owned(); + if prepared.get("policy_digest").and_then(Value::as_str) != Some(policy_digest) { + return Err(Reason::PolicyDigestMismatch.into()); + } + let unchanged = prepared + .get("local_ops") + .and_then(Value::as_array) + .is_some_and(Vec::is_empty) + && prepared.get("effective_action").and_then(Value::as_str) == Some("c2pa.opened"); + let geometry = prepared + .get_mut("geometry") + .map(Value::take) + .filter(|geometry| { + geometry.get("disposition").and_then(Value::as_str) == Some("external") + }); + drop(prepared); + let Some(geometry) = geometry.filter(|_| unchanged) else { + return Err(LargeRefusal::Lane("prepare_answer_invalid")); + }; + + // The probe recomputes the server's plan from the same bytes; an + // unchanged external plan hashes the whole source once. + let verified = match work + .race(probe.call( + "verify_plan", + &json!({ + "mime": mime, + "total_size": size, + "disclosure": disclosure, + "geometry": geometry, + }), + &pinned, + )) + .await + { + None => return Err(Reason::Timeout.into()), + Some(result) => result.map_err(|error| probe_refusal("verify_plan", error))?, + }; + let items = verified + .get("hash_items") + .and_then(Value::as_array) + .ok_or(LargeRefusal::Lane("probe_result_invalid"))?; + if !covers_whole_source(items, size) { + return Err(LargeRefusal::Lane("plan_not_whole_source")); + } + drop(verified); + match work.race(source.confirm_current()).await { + None => return Err(Reason::Timeout.into()), + Some(result) => result?, + } + + let finalize = json!({ + "page_url": job.reader_url, + "expected_policy_digest": policy_digest, + "protocol": PROTOCOL, + "plan_id": plan_id, + "binding": {"type": "data_hash", "digest": job.source_sha256}, + "xmp_reference": false, + }); + let manifest_limit = capabilities.manifest_limit(); + let answer_limit = manifest_limit + .saturating_mul(4) + .div_ceil(3) + .saturating_add(FINALIZE_OVERHEAD_BYTES); + let finalized = self + .post( + services, + FINALIZE_PATH, + &finalize, + &idempotency_key, + answer_limit, + work, + ) + .await + .map_err(|refusal| match refusal { + LargeRefusal::Module(Reason::ResponseTooLarge) => { + LargeRefusal::Lane(MANIFEST_TOO_LARGE) + } + other => other, + })?; + self.verify_finalized(&probe, &finalized, job, policy_digest, &capabilities, work) + .await + } + + /// Checks the finalize answer against the plan and the pinned source, + /// and that the store it returns binds the whole-source digest. + async fn verify_finalized( + &self, + probe: &ProbeModule, + finalized: &Value, + job: &LargeMediaJob<'_>, + policy_digest: &str, + capabilities: &Capabilities, + work: &mut Deadline<'_>, + ) -> Result { + let text = |name: &str| finalized.get(name).and_then(Value::as_str); + if text("policy_digest") != Some(policy_digest) { + return Err(Reason::PolicyDigestMismatch.into()); + } + let external = text("manifest_disposition") == Some("external") + && text("xmp_reference") == Some("declined") + && text("binding_algorithm") == Some("c2pa.hash.data") + && finalized.get("expected_final_size").and_then(Value::as_u64) + == Some(job.source.size()); + if !external { + return Err(LargeRefusal::Lane("finalize_answer_invalid")); + } + let store_b64 = text("manifest_store_b64").ok_or(Reason::UnsignedResponse)?; + let manifest_limit = capabilities.manifest_limit(); + if store_b64.len() + > manifest_limit + .saturating_mul(4) + .div_ceil(3) + .saturating_add(3) + { + return Err(LargeRefusal::Lane(MANIFEST_TOO_LARGE)); + } + let store = STANDARD + .decode(store_b64) + .map_err(|_| Reason::InvalidSignedResponse)?; + if store.is_empty() { + return Err(Reason::InvalidSignedResponse.into()); + } + if store.len() > manifest_limit { + return Err(LargeRefusal::Lane(MANIFEST_TOO_LARGE)); + } + if text("manifest_sha256") != Some(sha256_hex(&store).as_str()) { + return Err(Reason::SignedDigestMismatch.into()); + } + let manifest_url = text("manifest_url") + .filter(|url| valid_https_url(url)) + .ok_or(Reason::InvalidSignedResponse)?; + let verification_url = text("verification_url") + .filter(|url| valid_https_url(url)) + .ok_or(Reason::InvalidSignedResponse)?; + let instance_id = text("instance_id") + .filter(|id| valid_identifier(id)) + .ok_or(Reason::InvalidSignedResponse)?; + let document_id = text("document_id") + .filter(|id| valid_identifier(id)) + .ok_or(Reason::InvalidSignedResponse)?; + // The store's hard binding must be over the whole source, nothing + // excluded, with the digest core computed over the pinned version. + let checked = match work + .race(probe.call_without_source::( + "check_store", + &json!({ + "manifest_store_b64": store_b64, + "digest": job.source_sha256, + "binding_recipe": {"type": "data_hash", "algorithm": "sha256", "exclusions": []}, + }), + )) + .await + { + None => return Err(Reason::Timeout.into()), + Some(result) => result.map_err(|error| probe_refusal("check_store", error))?, + }; + if checked.get("ok").and_then(Value::as_bool) != Some(true) { + return Err(Reason::SignedDigestMismatch.into()); + } + Ok(Finalized { + manifest_url: manifest_url.to_owned(), + instance_id: instance_id.to_owned(), + document_id: document_id.to_owned(), + verification_url: verification_url.to_owned(), + }) + } + + /// The lane's capabilities for the asset at `page_url`. + async fn capabilities( + &self, + services: &RuntimeServices, + page_url: &str, + work: &mut Deadline<'_>, + ) -> Result { + let mut url = url::Url::parse(&format!( + "{}{CAPABILITIES_PATH}", + self.config.api_base_url.trim_end_matches('/') + )) + .map_err(|_| Reason::RequestBuild)?; + url.query_pairs_mut().append_pair("page_url", page_url); + let request = Request::builder() + .method(Method::GET) + .uri(url.as_str()) + .header(WORKER_TOKEN_HEADER, self.worker_token()?) + .body(EdgeBody::empty()) + .map_err(|_| Reason::RequestBuild)?; + let body = self + .exchange(services, request, MAX_CAPABILITIES_BYTES, work) + .await?; + Capabilities::parse(&body).ok_or(LargeRefusal::Lane("capabilities_invalid")) + } + + /// The newest verified release the service advertises, from the cache or + /// fetched, compiled with the edge's limits. + async fn load_probe( + &self, + services: &RuntimeServices, + authority: &ReleaseAuthority, + advertised: &[ProbeDescriptor], + work: &mut Deadline<'_>, + ) -> Result { + let descriptor = authority.select(advertised).map_err(|error| { + log::info!("encypher: large media probe release code={}", error.code()); + LargeRefusal::Lane(error.code()) + })?; + let kv = services.kv_store(); + let key = format!("{PROBE_KEY_PREFIX}:{}", descriptor.sha256); + let cached = match work.race(kv.get_bytes(&key)).await { + Some(Ok(Some(bytes))) if sha256_hex(&bytes) == descriptor.sha256 => Some(bytes), + _ => None, + }; + let artifact = match cached { + Some(bytes) => bytes, + None => { + let request = Request::builder() + .method(Method::GET) + .uri(format!( + "{}{PROBE_PATH}{}", + self.config.api_base_url.trim_end_matches('/'), + descriptor.artifact_path() + )) + .body(EdgeBody::empty()) + .map_err(|_| Reason::RequestBuild)?; + let bytes = self + .exchange(services, request, MAX_PROBE_BYTES, work) + .await?; + if sha256_hex(&bytes) != descriptor.sha256 { + return Err(LargeRefusal::Lane("PROBE_RELEASE_INVALID")); + } + // Releases never change, so a cached copy needs no expiry + // beyond the plan lifetime. + let ttl = Duration::from_secs(self.config.plan_ttl_seconds); + if !matches!( + work.race(kv.put_bytes_with_ttl(&key, bytes.clone(), ttl)) + .await, + Some(Ok(())) + ) { + log::debug!("encypher: probe release cache write failed"); + } + bytes + } + }; + let loaded = match work + // The host's default limits are the edge's: structure only, never + // the payload. A release that runs out of fuel is refused. + .race(ProbeModule::load( + authority, + descriptor, + &artifact, + ProbeLimits::default(), + )) + .await + { + None => return Err(Reason::Timeout.into()), + Some(result) => result, + }; + loaded.map_err(|error| { + log::info!("encypher: large media probe release code={}", error.code()); + LargeRefusal::Lane(error.code()) + }) + } + + /// One authenticated JSON `POST` to a prepared endpoint. + async fn post( + &self, + services: &RuntimeServices, + path: &str, + body: &impl Serialize, + idempotency_key: &str, + max_response_bytes: usize, + work: &mut Deadline<'_>, + ) -> Result { + let body = serde_json::to_vec(body).map_err(|_| Reason::RequestBuild)?; + let request = Request::builder() + .method(Method::POST) + .uri(format!( + "{}{path}", + self.config.api_base_url.trim_end_matches('/') + )) + .header(header::CONTENT_TYPE, "application/json") + .header(WORKER_TOKEN_HEADER, self.worker_token()?) + .header(IDEMPOTENCY_KEY_HEADER, idempotency_key) + .body(EdgeBody::from(body)) + .map_err(|_| Reason::RequestBuild)?; + let answer = self + .exchange(services, request, max_response_bytes, work) + .await?; + serde_json::from_slice(&answer).map_err(|_| Reason::MalformedResponse.into()) + } + + /// One bounded call; a non-`200` answer is a refusal with its code. + async fn exchange( + &self, + services: &RuntimeServices, + request: Request, + max_response_bytes: usize, + work: &mut Deadline<'_>, + ) -> Result { + let limit = max_response_bytes.saturating_add(1); + let response = self.send_api(services, request, limit, work).await?; + let status = response.status(); + if status != StatusCode::OK { + let code = work + .race( + response + .into_body() + .into_bytes_bounded(media_api::MAX_ERROR_BODY_BYTES), + ) + .await + .and_then(Result::ok) + .and_then(|body| media_api::error_code(&body)); + log::info!( + "encypher: large media answer status={} code={}", + status.as_u16(), + code.as_deref().unwrap_or("none") + ); + return Err(answer_refusal(status, code.as_deref())); + } + let body = match work + .race(response.into_body().into_bytes_bounded(limit)) + .await + { + None => return Err(Reason::Timeout.into()), + Some(Err(_)) => return Err(Reason::ResponseTooLarge.into()), + Some(Ok(body)) => body, + }; + if body.len() > max_response_bytes { + return Err(Reason::ResponseTooLarge.into()); + } + Ok(body) + } +} + +#[cfg(test)] +mod flow_tests; + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::Arc; + + use futures::executor::block_on; + use http::HeaderMap; + use trusted_server_core::media_signing::SourceVersion; + use trusted_server_core::platform::PlatformHttpClient; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + + use crate::EncypherConfig; + + const SHA: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + fn signer() -> Signer { + let config: EncypherConfig = serde_json::from_value(json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": "cwk_test_worker_token_value", + "max_large_media_bytes": 1_073_741_824_u64, + })) + .expect("should parse config"); + Signer::new(config, 5_000_000) + } + + /// The gate refusal for one job, made before any request is sent. + fn refusal( + reader_url: &str, + source_reference: Option<&str>, + reader_reference: Option<&str>, + origin_host: &str, + ) -> &'static str { + let http = Arc::new(StubHttpClient::new()); + let services = + build_services_with_http_client(Arc::clone(&http) as Arc); + let origin = url::Url::parse("https://origin.example.com/video/ferry.mp4") + .expect("should parse the origin URL"); + let headers = HeaderMap::new(); + let source = VersionedMediaSource::new( + &services, + &origin, + &headers, + "origin", + SourceVersion::new("\"v1\"", 64 * 1024 * 1024).expect("should pin a strong ETag"), + ); + let job = LargeMediaJob { + reader_url, + content_type: "video/mp4", + source: &source, + source_sha256: SHA, + source_reference, + reader_reference, + source_origin_host: origin_host, + remaining: Duration::from_secs(60), + }; + let code = block_on(signer().sign_large(&services, &job)) + .expect_err("should refuse at the gates") + .code(); + assert!( + http.recorded_request_methods().is_empty(), + "should refuse before any request" + ); + code + } + + #[test] + fn a_large_source_without_a_usable_reference_is_refused_before_any_request() { + let reader = "https://publisher.example.com/video/ferry.mp4"; + assert_eq!( + refusal( + "https://cdn.example.org/video/ferry.mp4", + None, + None, + "origin.example.com" + ), + "outside_claim" + ); + assert_eq!( + refusal(reader, None, None, "Origin.example.com:8080"), + "invalid_origin_host" + ); + // A fetched address on another host than the one fetched, and a + // reader address that is not the delivered one, name no source. + assert_eq!( + refusal( + reader, + Some("https://elsewhere.example.com/video/ferry.mp4"), + Some("https://publisher.example.com/video/other.mp4"), + "origin.example.com", + ), + "source_reference_required" + ); + assert_eq!( + refusal(reader, None, None, "origin.example.com"), + "source_reference_required" + ); + } + + #[test] + fn only_a_single_whole_source_range_is_an_unchanged_plan() { + assert!(covers_whole_source(&[json!({"range": [0, 100]})], 100)); + for items in [ + vec![json!({"range": [0, 99]})], + vec![json!({"range": [0, 50]}), json!({"range": [50, 50]})], + vec![json!({"range": [0, 100]}), json!({"literal": "QQ=="})], + vec![json!({"range": [0, 100], "skip": true})], + vec![], + ] { + assert!( + !covers_whole_source(&items, 100), + "should refuse {items:?} as an unchanged plan" + ); + } + } + + #[test] + fn a_source_with_its_own_manifest_is_reported_as_kept_not_failed_or_signed() { + let kept = answer_refusal( + StatusCode::UNPROCESSABLE_ENTITY, + Some("SOURCE_MANIFEST_PRESENT"), + ); + assert_eq!(kept.code(), "source_has_embedded_provenance"); + assert!( + kept.retains_provenance(), + "should report the provenance kept" + ); + for failed in [ + answer_refusal( + StatusCode::UNPROCESSABLE_ENTITY, + Some("HASH_MODE_UNSUPPORTED"), + ), + answer_refusal( + StatusCode::UNPROCESSABLE_ENTITY, + Some("SOURCE_PROVENANCE_INVALID"), + ), + LargeRefusal::Module(Reason::Timeout), + ] { + assert!( + !failed.retains_provenance(), + "{}: should not claim kept provenance", + failed.code() + ); + } + } + + #[test] + fn prepared_errors_map_to_specific_refusals() { + assert_eq!( + answer_refusal(StatusCode::CONFLICT, Some("TRUSTED_SERVER_POLICY_CHANGED")), + LargeRefusal::Module(Reason::PolicyChanged) + ); + assert_eq!( + answer_refusal(StatusCode::CONFLICT, Some("PROBE_VERSION_UNSUPPORTED")).code(), + "probe_release_stale" + ); + assert_eq!( + answer_refusal( + StatusCode::UNPROCESSABLE_ENTITY, + Some("SOURCE_PROVENANCE_INVALID") + ) + .code(), + "source_provenance_invalid" + ); + assert_eq!( + answer_refusal(StatusCode::FORBIDDEN, Some("claim_changed")), + LargeRefusal::Module(Reason::Forbidden) + ); + } + + #[test] + fn capabilities_parse_only_an_enabled_v2_lane() { + let body = |enabled: bool, protocol: &str| { + serde_json::to_vec(&json!({ + "protocol": protocol, + "enabled": enabled, + "policy_digest": "a".repeat(64), + "formats": ["video/mp4"], + "maximums": { + "total_size_bytes": 1_000_000_000_u64, + "structure_decoded_bytes": 33_554_432, + "structure_chunks": 8192, + "manifest_decoded_bytes": 1_048_576, + "prepare_request_bytes": 50_331_648, + }, + "probe": [], + })) + .expect("should encode") + }; + let parsed = Capabilities::parse(&body(true, PROTOCOL)).expect("should parse"); + assert_eq!(parsed.formats, ["video/mp4"]); + assert_eq!(parsed.manifest_decoded_bytes, 1_048_576); + assert!(Capabilities::parse(&body(false, PROTOCOL)).is_none()); + assert!(Capabilities::parse(&body(true, "encypher-prepared-signing/1")).is_none()); + } +} diff --git a/crates/provenance/encypher/src/processor/prepared/flow_tests.rs b/crates/provenance/encypher/src/processor/prepared/flow_tests.rs new file mode 100644 index 0000000000..8ae75e4d41 --- /dev/null +++ b/crates/provenance/encypher/src/processor/prepared/flow_tests.rs @@ -0,0 +1,581 @@ +//! The prepared lane end to end, against a stand-in probe release signed with +//! a test key, a stub service and a stub origin. +//! +//! The stand-in has no format logic: each operation answers JSON the test +//! chose, so these tests exercise the lane's own orchestration and checks, +//! and no Encypher planner code enters this repository. Production always +//! holds releases to the pinned production key; only these tests pass +//! another authority. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use ed25519_dalek::{Signer as _, SigningKey}; +use edgezero_core::key_value_store::KvPage; +use futures::executor::block_on; +use http::HeaderMap; +use serde_json::{Value, json}; +use trusted_server_core::media_signing::{ + LargeMediaJob, LargeSignedMedia, SourceVersion, VersionedMediaSource, +}; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use trusted_server_core::platform::{ + KvError, PlatformHttpClient, PlatformKvStore, RuntimeServices, +}; + +use super::{LargeRefusal, Signer}; +use crate::EncypherConfig; +use crate::media_api::sha256_hex; +use crate::prepared_probe::{ProbeDescriptor, ReleaseAuthority}; + +const POLICY: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const OTHER_POLICY: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const SOURCE_SHA: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; +const SIZE: u64 = 20_000_000; +const READER_URL: &str = "https://publisher.example.com/video/ferry.mp4"; +const ORIGIN_HOST: &str = "publisher.example.com"; +const PLAN_ID: &str = "0f8fad5b-d9cb-469f-a165-70867728950e"; +const STORE: &[u8] = b"jumb-c2pa-remote-manifest-store"; + +const STAND_IN: &str = r#" +(module + (import "env" "host_read" (func $host_read (param i64 i32 i32) (result i32))) + (memory (export "memory") 2) + (global $heap (mut i32) (i32.const 16384)) + (global $result_ptr (mut i32) (i32.const 0)) + (global $result_len (mut i32) (i32.const 0)) + (data (i32.const 1024) "@VERSION@") + (data (i32.const 2048) "@EXTRACT@") + (data (i32.const 4096) "@VERIFY@") + (data (i32.const 6144) "@CHECK@") + (data (i32.const 8192) "@UNKNOWN@") + (func (export "probe_abi") (result i32) (i32.const 1)) + (func (export "probe_alloc") (param $length i32) (result i32) + (local $pointer i32) + (local.set $pointer (global.get $heap)) + (global.set $heap (i32.add (global.get $heap) (local.get $length))) + (local.get $pointer)) + (func (export "probe_free") (param i32 i32)) + (func (export "probe_result_ptr") (result i32) (global.get $result_ptr)) + (func (export "probe_result_len") (result i32) (global.get $result_len)) + (func $answer (param $pointer i32) (param $length i32) (param $code i32) (result i32) + (global.set $result_ptr (local.get $pointer)) + (global.set $result_len (local.get $length)) + (local.get $code)) + (func (export "probe_call") + (param $op i32) (param $op_len i32) (param $arg i32) (param $arg_len i32) (result i32) + (local $kind i32) + (local.set $kind (i32.load8_u (local.get $op))) + ;; "version" (7 bytes) and "verify_plan" (11 bytes) both start with v + (if (i32.and (i32.eq (local.get $kind) (i32.const 118)) (i32.eq (local.get $op_len) (i32.const 7))) + (then (return (call $answer (i32.const 1024) (i32.const @VERSION_LEN@) (i32.const 0))))) + (if (i32.eq (local.get $kind) (i32.const 118)) + (then (return (call $answer (i32.const 4096) (i32.const @VERIFY_LEN@) (i32.const 0))))) + ;; "extract" + (if (i32.eq (local.get $kind) (i32.const 101)) + (then (return (call $answer (i32.const 2048) (i32.const @EXTRACT_LEN@) (i32.const 0))))) + ;; "check_store" + (if (i32.eq (local.get $kind) (i32.const 99)) + (then (return (call $answer (i32.const 6144) (i32.const @CHECK_LEN@) (i32.const 0))))) + (call $answer (i32.const 8192) (i32.const @UNKNOWN_LEN@) (i32.const 1))) +) +"#; + +/// What each stand-in operation answers. +struct Answers { + extract: Value, + verify: Value, + check: Value, +} + +impl Default for Answers { + fn default() -> Self { + Self { + extract: json!({ + "chunks": [{"offset": 0, "data": "AAAA"}], + "has_provenance_reference": false, + }), + verify: json!({"hash_items": [{"range": [0, SIZE]}]}), + check: json!({"ok": true}), + } + } +} + +fn planner() -> String { + "d".repeat(64) +} + +fn wat_string(text: &str) -> String { + text.replace('\\', "\\\\").replace('"', "\\\"") +} + +/// The stand-in release answering `answers`. +fn release(answers: &Answers) -> Vec { + let version = json!({"planner_id": planner()}).to_string(); + let unknown = + json!({"error": {"code": "UNKNOWN_OP", "integrity": false, "message": "no such op"}}) + .to_string(); + let extract = answers.extract.to_string(); + let verify = answers.verify.to_string(); + let check = answers.check.to_string(); + let text = STAND_IN + .replace("@VERSION@", &wat_string(&version)) + .replace("@VERSION_LEN@", &version.len().to_string()) + .replace("@EXTRACT@", &wat_string(&extract)) + .replace("@EXTRACT_LEN@", &extract.len().to_string()) + .replace("@VERIFY@", &wat_string(&verify)) + .replace("@VERIFY_LEN@", &verify.len().to_string()) + .replace("@CHECK@", &wat_string(&check)) + .replace("@CHECK_LEN@", &check.len().to_string()) + .replace("@UNKNOWN@", &wat_string(&unknown)) + .replace("@UNKNOWN_LEN@", &unknown.len().to_string()); + wat::parse_str(&text).expect("should assemble the stand-in release") +} + +fn release_key() -> SigningKey { + SigningKey::from_bytes(&[11; 32]) +} + +/// The descriptor of `artifact`, signed with the test release key. +fn descriptor(artifact: &[u8]) -> ProbeDescriptor { + let mut descriptor = ProbeDescriptor { + abi: 1, + planner_id: planner(), + release: 3, + sha256: sha256_hex(artifact), + signature: String::new(), + }; + let signature = release_key().sign(descriptor.canonical_json().as_bytes()); + descriptor.signature = STANDARD.encode(signature.to_bytes()); + descriptor +} + +fn test_authority() -> ReleaseAuthority { + ReleaseAuthority::pinned(vec![release_key().verifying_key()], 1, Vec::new()) +} + +/// A runtime KV store held in memory. +#[derive(Default)] +struct MemoryKv(Mutex>); + +impl MemoryKv { + fn has(&self, key: &str) -> bool { + self.0.lock().expect("should lock").contains_key(key) + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + Ok(self.0.lock().expect("should lock").get(key).cloned()) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.0 + .lock() + .expect("should lock") + .insert(key.to_owned(), value); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + _ttl: std::time::Duration, + ) -> Result<(), KvError> { + self.put_bytes(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.0.lock().expect("should lock").remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(KvPage::default()) + } +} + +fn signer() -> Signer { + let config: EncypherConfig = serde_json::from_value(json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": "cwk_test_worker_token_value", + "max_large_media_bytes": 1_073_741_824_u64, + })) + .expect("should parse config"); + Signer::new(config, 5_000_000) +} + +fn finalize_answer() -> Value { + json!({ + "protocol": "encypher-prepared-signing/2", + "plan_id": PLAN_ID, + "instance_id": "xmp:iid:0001", + "document_id": "doc-1", + "manifest_disposition": "external", + "content_hash_kind": "hard_binding", + "binding_algorithm": "c2pa.hash.data", + "manifest_store_b64": STANDARD.encode(STORE), + "manifest_sha256": sha256_hex(STORE), + "manifest_url": "https://api.example.com/manifests/doc-1", + "xmp_reference": "declined", + "expected_final_size": SIZE, + "verification_url": "https://verify.example.com/doc-1", + "reused": false, + "policy_digest": POLICY, + }) +} + +fn prepare_answer() -> Value { + json!({ + "protocol": "encypher-prepared-signing/2", + "plan_id": PLAN_ID, + "instance_id": "xmp:iid:0001", + "document_id": "doc-1", + "mime": "video/mp4", + "effective_action": "c2pa.opened", + "geometry": {"disposition": "external"}, + "local_ops": [], + "policy_digest": POLICY, + }) +} + +/// One run of the lane, and what it asked of the service and origin. +struct Run { + result: Result, + http: Arc, + kv: Arc, + signer: Signer, +} + +impl Run { + fn methods(&self) -> Vec { + self.http.recorded_request_methods() + } + + fn refusal(&self) -> LargeRefusal { + *self.result.as_ref().expect_err("should refuse") + } + + fn body(&self, index: usize) -> Value { + serde_json::from_slice(&self.http.recorded_request_bodies()[index]) + .expect("should send JSON") + } +} + +/// The scenario's answers, queued in the order the lane asks for them. +struct Scenario { + answers: Answers, + capabilities_policy: &'static str, + /// The largest manifest store the service says it returns. + manifest_decoded_bytes: usize, + prepare: Value, + finalize: Value, +} + +impl Default for Scenario { + fn default() -> Self { + Self { + answers: Answers::default(), + capabilities_policy: POLICY, + manifest_decoded_bytes: 1_048_576, + prepare: prepare_answer(), + finalize: finalize_answer(), + } + } +} + +fn run(scenario: &Scenario) -> Run { + let artifact = release(&scenario.answers); + let probe = descriptor(&artifact); + let http = Arc::new(StubHttpClient::new()); + http.set_streaming_responses_supported(true); + http.push_response( + 200, + serde_json::to_vec(&json!({"policy_digest": POLICY})).expect("JSON"), + ); + http.push_response( + 200, + serde_json::to_vec(&json!({ + "protocol": "encypher-prepared-signing/2", + "enabled": true, + "policy_digest": scenario.capabilities_policy, + "formats": ["video/mp4"], + "maximums": { + "total_size_bytes": 9_007_199_246_352_383_u64, + "structure_decoded_bytes": 33_554_432, + "structure_chunks": 8192, + "manifest_decoded_bytes": scenario.manifest_decoded_bytes, + "prepare_request_bytes": 50_331_648, + }, + "probe": [probe.to_json()], + })) + .expect("JSON"), + ); + http.push_response(200, artifact); + http.push_response(200, serde_json::to_vec(&scenario.prepare).expect("JSON")); + // The pinned one-byte read that confirms the version before finalize. + http.push_response_with_headers( + 206, + b"\x00".to_vec(), + vec![ + ("content-range", format!("bytes 0-0/{SIZE}")), + ("content-length", "1".to_owned()), + ("etag", "\"v1\"".to_owned()), + ], + ); + http.push_response(200, serde_json::to_vec(&scenario.finalize).expect("JSON")); + + let kv = Arc::new(MemoryKv::default()); + let services: RuntimeServices = + build_services_with_http_client(Arc::clone(&http) as Arc) + .with_kv_store(Arc::clone(&kv) as Arc); + let origin = url::Url::parse("https://publisher.example.com/video/ferry.mp4") + .expect("should parse the origin URL"); + let headers = HeaderMap::new(); + let source = VersionedMediaSource::new( + &services, + &origin, + &headers, + "origin", + SourceVersion::new("\"v1\"", SIZE).expect("should pin a strong ETag"), + ); + let job = LargeMediaJob { + reader_url: READER_URL, + content_type: "video/mp4", + source: &source, + source_sha256: SOURCE_SHA, + source_reference: Some(READER_URL), + reader_reference: None, + source_origin_host: ORIGIN_HOST, + remaining: std::time::Duration::from_secs(60), + }; + let signer = signer(); + let result = block_on(signer.sign_large_with(&services, &job, &test_authority())); + Run { + result, + http, + kv, + signer, + } +} + +#[test] +fn an_unchanged_source_is_bound_whole_after_its_version_is_confirmed() { + let run = run(&Scenario::default()); + + run.result.as_ref().expect("should sign"); + assert_eq!( + run.methods(), + ["GET", "GET", "GET", "POST", "GET", "POST"], + "should confirm the pinned version after prepare and before finalize" + ); + let finalize = run.body(5); + assert_eq!( + finalize["binding"], + json!({"type": "data_hash", "digest": SOURCE_SHA}), + "should bind the whole-source digest" + ); + assert_eq!( + finalize["xmp_reference"], false, + "should never edit the source" + ); + let keys: Vec = run + .http + .recorded_request_headers() + .iter() + .filter_map(|headers| { + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("idempotency-key")) + .map(|(_, value)| value.clone()) + }) + .collect(); + assert_eq!(keys.len(), 2, "should key prepare and finalize"); + assert_eq!(keys[0], keys[1], "should finalize under the plan's own key"); +} + +#[test] +fn a_manifest_larger_than_the_lane_takes_is_refused() { + let run = run(&Scenario { + manifest_decoded_bytes: STORE.len() - 1, + ..Scenario::default() + }); + assert_eq!( + run.refusal().code(), + "manifest_too_large", + "should hold the store to the smaller of the service's and the edge's limit" + ); +} + +#[test] +fn a_source_with_provenance_of_its_own_keeps_it_and_is_never_prepared() { + for (extract, reason) in [ + ( + json!({ + "chunks": [{"offset": 0, "data": "AAAA"}], + "credentials": {"manifests": 1}, + "has_provenance_reference": false, + }), + "source_has_embedded_provenance", + ), + ( + json!({ + "chunks": [{"offset": 0, "data": "AAAA"}], + "has_provenance_reference": true, + }), + "source_has_provenance_reference", + ), + ] { + let run = run(&Scenario { + answers: Answers { + extract, + ..Answers::default() + }, + ..Scenario::default() + }); + + let refusal = run.refusal(); + assert_eq!(refusal.code(), reason); + assert!(refusal.retains_provenance(), "{reason}: should keep it"); + assert!( + !run.methods().iter().any(|method| method == "POST"), + "{reason}: should never prepare or finalize" + ); + } +} + +#[test] +fn a_release_that_does_not_report_a_provenance_reference_is_not_trusted() { + for flag in [None, Some(json!("false")), Some(json!(0))] { + let mut extract = json!({"chunks": [{"offset": 0, "data": "AAAA"}]}); + if let Some(flag) = flag.clone() { + extract["has_provenance_reference"] = flag; + } + let run = run(&Scenario { + answers: Answers { + extract, + ..Answers::default() + }, + ..Scenario::default() + }); + + let refusal = run.refusal(); + assert_eq!(refusal.code(), "probe_result_invalid", "{flag:?}"); + assert!(!refusal.retains_provenance(), "{flag:?}: should not claim"); + assert!( + !run.methods().iter().any(|method| method == "POST"), + "{flag:?}: should never prepare" + ); + } +} + +#[test] +fn a_plan_that_is_not_the_unchanged_whole_source_is_never_finalized() { + let mut edits = prepare_answer(); + edits["local_ops"] = json!([{"op": "xmp"}]); + let mut edited = prepare_answer(); + edited["effective_action"] = json!("c2pa.edited"); + let mut embedded = prepare_answer(); + embedded["geometry"] = json!({"disposition": "embedded"}); + for prepare in [edits, edited, embedded] { + let run = run(&Scenario { + prepare, + ..Scenario::default() + }); + assert_eq!(run.refusal().code(), "prepare_answer_invalid"); + assert_eq!(run.methods().len(), 4, "should stop after prepare"); + } + + let run = run(&Scenario { + answers: Answers { + verify: json!({"hash_items": [{"range": [0, SIZE - 1]}]}), + ..Answers::default() + }, + ..Scenario::default() + }); + assert_eq!(run.refusal().code(), "plan_not_whole_source"); + assert_eq!( + run.methods().len(), + 4, + "should stop before the version check" + ); +} + +#[test] +fn a_finalized_answer_that_does_not_bind_the_source_is_never_delivered() { + let mut resized = finalize_answer(); + resized["expected_final_size"] = json!(SIZE + 1); + let mut swapped = finalize_answer(); + swapped["manifest_sha256"] = json!(sha256_hex(b"another store")); + let mut embedded = finalize_answer(); + embedded["manifest_disposition"] = json!("embedded"); + for (finalize, reason) in [ + (resized, "finalize_answer_invalid"), + (swapped, "signed_digest_mismatch"), + (embedded, "finalize_answer_invalid"), + ] { + let run = run(&Scenario { + finalize, + ..Scenario::default() + }); + assert_eq!(run.refusal().code(), reason); + } + + let run = run(&Scenario { + answers: Answers { + check: json!({"ok": false}), + ..Answers::default() + }, + ..Scenario::default() + }); + assert_eq!( + run.refusal().code(), + "signed_digest_mismatch", + "should refuse a store that does not bind the whole-source digest" + ); +} + +#[test] +fn a_changed_policy_evicts_the_cached_policy_and_publishes_nothing() { + let changed = run(&Scenario { + capabilities_policy: OTHER_POLICY, + ..Scenario::default() + }); + assert_eq!(changed.refusal().code(), "policy_changed"); + assert!( + !changed.kv.has(&changed.signer.policy_key), + "should evict the policy the lane held" + ); + assert!(!changed.methods().iter().any(|method| method == "POST")); + + let mut prepared = prepare_answer(); + prepared["policy_digest"] = json!(OTHER_POLICY); + let mismatched = run(&Scenario { + prepare: prepared, + ..Scenario::default() + }); + assert_eq!(mismatched.refusal().code(), "policy_digest_mismatch"); + assert!(!mismatched.kv.has(&mismatched.signer.policy_key)); + + let mut finalized = finalize_answer(); + finalized["policy_digest"] = json!(OTHER_POLICY); + let late = run(&Scenario { + finalize: finalized, + ..Scenario::default() + }); + assert_eq!(late.refusal().code(), "policy_digest_mismatch"); + assert!(!late.kv.has(&late.signer.policy_key)); +} diff --git a/crates/provenance/encypher/src/svg_equivalence.rs b/crates/provenance/encypher/src/svg_equivalence.rs new file mode 100644 index 0000000000..3f4a9d0685 --- /dev/null +++ b/crates/provenance/encypher/src/svg_equivalence.rs @@ -0,0 +1,977 @@ +//! The standalone SVG document an inline `` is signed as. +//! +//! A browser builds an inline SVG's tree with the HTML parser, which reads +//! it differently from an XML parser: it needs no namespace declaration, +//! keeps the first of two attributes whose names differ only in case, and +//! decodes HTML character references. What is signed must be a standalone +//! SVG document whose XML tree is the tree the browser built, so a verifier +//! reading the signed bytes sees what the reader saw. +//! +//! The fragment is parsed as HTML in a `body` context, into scraper's +//! namespace-aware tree, as the browser parsed it, and refused unless that +//! yields exactly one `` root with nothing displaced beside it but +//! whitespace. Where the original bytes already parse as XML to the same +//! tree, they are signed unchanged. Otherwise, unless the fragment carries +//! provenance of its own, which is never rewritten, the browser's tree is +//! serialized as XML and kept only when that serialization parses to the +//! same tree both as HTML and as XML. +//! +//! Trees are compared by element namespace and local name, the set of +//! ordinary attributes by namespace, local name and value, text with +//! adjacent text merged and CDATA read as text, and comments. Namespace +//! declarations are not ordinary attributes and are not compared: only the +//! namespaces they resolve are. Anything else, a processing instruction or a +//! doctype, makes two trees unequal. XML is always parsed with DTDs +//! refused, so no entity is ever declared or resolved. + +use std::borrow::Cow; +use std::marker::PhantomData; + +use ego_tree::NodeRef; +use ego_tree::iter::Children; +use html5ever::tendril::TendrilSink as _; +use html5ever::{LocalName, Namespace, ParseOpts, QualName, parse_fragment}; +use scraper::node::Element; +use scraper::{HtmlTreeSink, Node}; + +use crate::embedded; +use crate::processor::Reason; + +const HTML_NAMESPACE: &str = "http://www.w3.org/1999/xhtml"; +const SVG_NAMESPACE: &str = "http://www.w3.org/2000/svg"; +const XLINK_NAMESPACE: &str = "http://www.w3.org/1999/xlink"; +const XML_NAMESPACE: &str = "http://www.w3.org/XML/1998/namespace"; +const XMLNS_NAMESPACE: &str = "http://www.w3.org/2000/xmlns/"; + +/// The most nodes an inline SVG may hold. +const MAX_NODES: usize = 10_000; +/// The deepest an inline SVG's elements may nest. +const MAX_DEPTH: usize = 256; +/// Nodes the HTML fragment parser makes besides the fragment's own: the +/// fragment root, the context element and the root `html` element. +const PARSER_NODES: usize = 3; + +/// The declarations the serialization's root carries. +const SVG_DECLARATION: &str = " xmlns=\"http://www.w3.org/2000/svg\""; +const XLINK_DECLARATION: &str = " xmlns:xlink=\"http://www.w3.org/1999/xlink\""; + +/// The standalone SVG document `fragment`, an inline `` element as +/// written in an HTML page, is signed as: `fragment` itself where it +/// already parses as XML to the tree the browser built from it, else that +/// tree serialized as XML. +/// +/// # Errors +/// +/// - [`Reason::MediaTooLarge`] where `fragment` is longer than `max_bytes`. +/// - [`Reason::SvgProvenanceNotEquivalent`] where `fragment` carries +/// provenance of its own and is not, as written, equivalent: it is never +/// rewritten. +/// - [`Reason::InlineSvgNotEquivalent`] where the browser's tree is not one +/// `` root, exceeds the node or depth bound, or has no XML +/// serialization that parses back to it as both HTML and XML. +pub(crate) fn signable_svg(fragment: &str, max_bytes: usize) -> Result, Reason> { + if fragment.len() > max_bytes { + return Err(Reason::MediaTooLarge); + } + let parsed = parse_html(fragment).ok_or(Reason::InlineSvgNotEquivalent)?; + let root = parsed.root().ok_or(Reason::InlineSvgNotEquivalent)?; + if let Some(document) = parse_xml(fragment) + && equal(&Browser, root, &Xml(PhantomData), document.root_element()) + { + return Ok(Cow::Borrowed(fragment)); + } + if carries_provenance(fragment, root) { + return Err(Reason::SvgProvenanceNotEquivalent); + } + let serialized = serialize(root, max_bytes).ok_or(Reason::InlineSvgNotEquivalent)?; + let equivalent = { + let as_html = parse_html(&serialized); + let as_xml = parse_xml(&serialized); + match (as_html.as_ref().and_then(Parsed::root), as_xml) { + (Some(as_html), Some(as_xml)) => { + equal(&Browser, root, &Browser, as_html) + && equal(&Browser, root, &Xml(PhantomData), as_xml.root_element()) + } + _ => false, + } + }; + if equivalent { + Ok(Cow::Owned(serialized)) + } else { + Err(Reason::InlineSvgNotEquivalent) + } +} + +/// Refuse all edits inside an original inline SVG that carries provenance. +/// A fragment whose structure cannot be inspected within the parser's limits +/// is also left intact, rather than assuming it has no provenance. +pub(crate) fn descendant_edit_refusal(fragment: &str) -> Option { + let Some(parsed) = parse_html(fragment) else { + return Some(Reason::InlineSvgNotEquivalent); + }; + let Some(root) = parsed.root() else { + return Some(Reason::InlineSvgNotEquivalent); + }; + carries_provenance(fragment, root).then_some(Reason::SvgExistingProvenance) +} + +/// Whether the inline SVG under `root` carries provenance of its own: the +/// manifest carrier the signer's writer finds, an element or attribute in +/// the `c2pa` prefix, a declaration of a C2PA namespace, or a provenance +/// reference such as XMP's `dcterms:provenance`. The HTML parser keeps such +/// prefixed names whole, as written. Ordinary text that mentions C2PA is +/// not provenance. +fn carries_provenance(fragment: &str, root: NodeRef<'_, Node>) -> bool { + if embedded::existing_carrier(fragment.as_bytes()).is_some() { + return true; + } + root.descendants().any(|node| { + let Node::Element(element) = node.value() else { + return false; + }; + names_provenance(&element.name.local) + || element.attrs.iter().any(|(name, value)| { + names_provenance(&name.local) + || (is_declaration(&name.ns, &name.local) && is_c2pa_namespace(value)) + }) + }) +} + +/// Whether a name the HTML parser kept whole, prefix included, is in the +/// `c2pa` prefix or names a provenance reference. +fn names_provenance(name: &str) -> bool { + let (prefix, local) = name.rsplit_once(':').unwrap_or(("", name)); + prefix.eq_ignore_ascii_case("c2pa") || local.eq_ignore_ascii_case("provenance") +} + +fn is_c2pa_namespace(uri: &str) -> bool { + let uri = uri.trim(); + ["http://c2pa.org/", "https://c2pa.org/"] + .iter() + .any(|start| { + uri.get(..start.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(start)) + }) +} + +/// scraper's tree for one fragment, with its `` root. +struct Parsed { + html: scraper::Html, + root: ego_tree::NodeId, +} + +impl Parsed { + fn root(&self) -> Option> { + self.html.tree.get(self.root) + } +} + +/// `fragment` parsed as HTML in a `body` context with scripting on, as a +/// browser parses inline SVG, into scraper's tree, or `None` unless that +/// yields exactly one `` element at the top, with only whitespace text +/// beside it, within the node bound. +fn parse_html(fragment: &str) -> Option { + // Every element and comment the tokenizer starts begins at a `<`, so + // this bounds what the parse allocates before the exact count below. + if fragment.bytes().filter(|byte| *byte == b'<').count() > MAX_NODES { + return None; + } + let context = QualName::new( + None, + Namespace::from(HTML_NAMESPACE), + LocalName::from("body"), + ); + let html = parse_fragment( + HtmlTreeSink::new(scraper::Html::new_fragment()), + ParseOpts::default(), + context, + Vec::new(), + true, + ) + .one(fragment); + if html.tree.values().count() > MAX_NODES + PARSER_NODES { + return None; + } + let top = html + .tree + .root() + .children() + .find(|child| child.value().is_element())?; + let mut root = None; + for child in top.children() { + match child.value() { + Node::Element(element) + if root.is_none() + && &*element.name.ns == SVG_NAMESPACE + && &*element.name.local == "svg" => + { + root = Some(child.id()); + } + Node::Text(text) if text.chars().all(is_html_whitespace) => {} + _ => return None, + } + } + let root = root?; + Some(Parsed { html, root }) +} + +fn is_html_whitespace(character: char) -> bool { + matches!(character, '\t' | '\n' | '\u{c}' | '\r' | ' ') +} + +/// `text` parsed as XML with DTDs refused, so no entity is declared or +/// resolved, or `None` where it is not well-formed. +fn parse_xml(text: &str) -> Option> { + let options = roxmltree::ParsingOptions { + allow_dtd: false, + nodes_limit: u32::try_from(MAX_NODES + 1).unwrap_or(u32::MAX), + ..roxmltree::ParsingOptions::default() + }; + roxmltree::Document::parse_with_options(text, options).ok() +} + +/// What a compared node is. +enum Kind<'a> { + Element { namespace: &'a str, name: &'a str }, + Text(&'a str), + Comment(&'a str), + Other, +} + +/// A tree, as the comparison walks it. +trait View<'a> { + type Node: Copy; + type Children: Iterator; + + fn kind(&self, node: Self::Node) -> Kind<'a>; + /// The element's ordinary attributes as (namespace, local name, value), + /// sorted, namespace declarations excluded. + fn attributes(&self, node: Self::Node) -> Vec<(&'a str, &'a str, &'a str)>; + fn children(&self, node: Self::Node) -> Self::Children; +} + +/// Whether an attribute is a namespace declaration rather than an ordinary +/// attribute. The HTML parser puts `xmlns` and `xmlns:xlink` in the XMLNS +/// namespace and leaves any other `xmlns:` prefixed name unnamespaced. +fn is_declaration(namespace: &str, name: &str) -> bool { + namespace == XMLNS_NAMESPACE + || (namespace.is_empty() && (name == "xmlns" || name.starts_with("xmlns:"))) +} + +/// scraper's tree, whose node references carry the tree. +struct Browser; + +impl<'a> View<'a> for Browser { + type Node = NodeRef<'a, Node>; + type Children = Children<'a, Node>; + + fn kind(&self, node: Self::Node) -> Kind<'a> { + match node.value() { + Node::Element(element) => Kind::Element { + namespace: &element.name.ns, + name: &element.name.local, + }, + Node::Text(text) => Kind::Text(&text.text), + Node::Comment(comment) => Kind::Comment(&comment.comment), + _ => Kind::Other, + } + } + + fn attributes(&self, node: Self::Node) -> Vec<(&'a str, &'a str, &'a str)> { + let Node::Element(element) = node.value() else { + return Vec::new(); + }; + let mut ordinary: Vec<(&'a str, &'a str, &'a str)> = element + .attrs + .iter() + .map(|(name, value)| (&*name.ns, &*name.local, &**value)) + .filter(|(namespace, name, _)| !is_declaration(namespace, name)) + .collect(); + ordinary.sort_unstable(); + ordinary + } + + fn children(&self, node: Self::Node) -> Self::Children { + node.children() + } +} + +/// A roxmltree document, whose nodes carry their own references. +struct Xml<'a, 'input: 'a>(PhantomData>); + +impl<'a, 'input: 'a> View<'a> for Xml<'a, 'input> { + type Node = roxmltree::Node<'a, 'input>; + type Children = roxmltree::Children<'a, 'input>; + + fn kind(&self, node: Self::Node) -> Kind<'a> { + match node.node_type() { + roxmltree::NodeType::Element => { + let name = node.tag_name(); + Kind::Element { + namespace: name.namespace().unwrap_or_default(), + name: name.name(), + } + } + roxmltree::NodeType::Text => Kind::Text(node.text().unwrap_or_default()), + roxmltree::NodeType::Comment => Kind::Comment(node.text().unwrap_or_default()), + roxmltree::NodeType::Root | roxmltree::NodeType::PI => Kind::Other, + } + } + + fn attributes(&self, node: Self::Node) -> Vec<(&'a str, &'a str, &'a str)> { + // roxmltree keeps namespace declarations apart from attributes. + let mut ordinary: Vec<_> = node + .attributes() + .map(|attribute| { + ( + attribute.namespace().unwrap_or_default(), + attribute.name(), + attribute.value(), + ) + }) + .collect(); + ordinary.sort_unstable(); + ordinary + } + + fn children(&self, node: Self::Node) -> Self::Children { + node.children() + } +} + +/// Whether the trees under `a_root` and `b_root` are equal, within the +/// depth bound. The walk keeps one pair of child iterators per open element +/// and copies neither tree. +fn equal<'a, 'b, A: View<'a>, B: View<'b>>(a: &A, a_root: A::Node, b: &B, b_root: B::Node) -> bool { + if !same(a, a_root, b, b_root) { + return false; + } + let mut open = vec![(a.children(a_root), b.children(b_root))]; + while let Some((mut a_children, mut b_children)) = open.pop() { + match (a_children.next(), b_children.next()) { + (None, None) => {} + (Some(a_node), Some(b_node)) => { + if !same(a, a_node, b, b_node) { + return false; + } + let element = matches!(a.kind(a_node), Kind::Element { .. }); + open.push((a_children, b_children)); + if element { + if open.len() >= MAX_DEPTH { + return false; + } + open.push((a.children(a_node), b.children(b_node))); + } + } + _ => return false, + } + } + true +} + +fn same<'a, 'b, A: View<'a>, B: View<'b>>(a: &A, a_node: A::Node, b: &B, b_node: B::Node) -> bool { + match (a.kind(a_node), b.kind(b_node)) { + ( + Kind::Element { + namespace: a_namespace, + name: a_name, + }, + Kind::Element { + namespace: b_namespace, + name: b_name, + }, + ) => { + a_namespace == b_namespace + && a_name == b_name + && a.attributes(a_node) == b.attributes(b_node) + } + (Kind::Text(a_text), Kind::Text(b_text)) + | (Kind::Comment(a_text), Kind::Comment(b_text)) => a_text == b_text, + _ => false, + } +} + +/// The tree under `root` written as a standalone SVG document, or `None` +/// where XML cannot carry it: an element outside the SVG namespace, an +/// attribute outside no namespace, `XLink` or XML, a name XML cannot spell, a +/// character XML forbids, a comment XML cannot hold, any other node, or +/// more than `max_bytes`. +/// +/// The root declares the SVG namespace, and the `XLink` prefix where an +/// attribute uses it. Every element and ordinary attribute is the tree's, +/// with its value, which is the HTML parser's: the first of attributes that +/// differ only in case, and character references decoded. Values escape +/// `&`, `<`, `>`, `"`, tab, line feed and carriage return, and text escapes +/// `&`, `<`, `>` and carriage return, so XML's attribute-value and line-end +/// normalization leaves them as parsed. +fn serialize(root: NodeRef<'_, Node>, max_bytes: usize) -> Option { + let Node::Element(element) = root.value() else { + return None; + }; + let mut output = SvgOutput::new(max_bytes); + let mut uses_xlink = false; + write_start(&mut output, element, &mut uses_xlink)?; + // The root's declarations follow its name: the XLink one is known only + // once every attribute is written. + let declarations = 1 + element.name.local.len(); + output.insert_str(declarations, SVG_DECLARATION)?; + let mut open: Vec<(Children<'_, Node>, &str)> = Vec::new(); + if root.first_child().is_some() { + output.push('>')?; + open.push((root.children(), &*element.name.local)); + } else { + output.push_str("/>")?; + } + while let Some((mut children, name)) = open.pop() { + let Some(child) = children.next() else { + output.push_str("')?; + continue; + }; + open.push((children, name)); + match child.value() { + Node::Element(element) => { + write_start(&mut output, element, &mut uses_xlink)?; + if child.first_child().is_some() { + if open.len() >= MAX_DEPTH { + return None; + } + output.push('>')?; + open.push((child.children(), &*element.name.local)); + } else { + output.push_str("/>")?; + } + } + Node::Text(text) => write_text(&mut output, &text.text)?, + Node::Comment(comment) => write_comment(&mut output, &comment.comment)?, + _ => return None, + } + } + if uses_xlink { + output.insert_str(declarations + SVG_DECLARATION.len(), XLINK_DECLARATION)?; + } + Some(output.text) +} + +/// A serializer buffer whose length and requested allocation never exceed +/// the output limit. Grow geometrically, but request the exact bounded target +/// so String's implicit doubling cannot allocate beyond it. +struct SvgOutput { + text: String, + max_bytes: usize, +} + +impl SvgOutput { + fn new(max_bytes: usize) -> Self { + Self { + text: String::new(), + max_bytes, + } + } + + fn reserve(&mut self, additional: usize) -> Option<()> { + let needed = self.text.len().checked_add(additional)?; + if needed > self.max_bytes { + return None; + } + if needed > self.text.capacity() { + let target = self + .text + .capacity() + .checked_mul(2) + .unwrap_or(self.max_bytes) + .max(64.min(self.max_bytes)) + .max(needed) + .min(self.max_bytes); + self.text + .try_reserve_exact(target.checked_sub(self.text.len())?) + .ok()?; + } + Some(()) + } + + fn push_str(&mut self, text: &str) -> Option<()> { + self.reserve(text.len())?; + self.text.push_str(text); + Some(()) + } + + fn push(&mut self, character: char) -> Option<()> { + self.reserve(character.len_utf8())?; + self.text.push(character); + Some(()) + } + + fn insert_str(&mut self, at: usize, text: &str) -> Option<()> { + self.reserve(text.len())?; + self.text.insert_str(at, text); + Some(()) + } +} + +/// Writes ` Option<()> { + if &*element.name.ns != SVG_NAMESPACE || !is_xml_local_name(&element.name.local) { + return None; + } + output.push('<')?; + output.push_str(&element.name.local)?; + for (name, value) in element.attrs.iter() { + let namespace = &*name.ns; + let local = &*name.local; + if is_declaration(namespace, local) { + continue; + } + let prefix = match namespace { + "" => "", + XLINK_NAMESPACE => { + *uses_xlink = true; + "xlink:" + } + XML_NAMESPACE => "xml:", + _ => return None, + }; + if !is_xml_local_name(local) { + return None; + } + output.push(' ')?; + output.push_str(prefix)?; + output.push_str(local)?; + output.push_str("=\"")?; + for character in value.chars() { + match character { + '&' => output.push_str("&"), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + '"' => output.push_str("""), + '\t' => output.push_str(" "), + '\n' => output.push_str(" "), + '\r' => output.push_str(" "), + character if is_xml_char(character) => output.push(character), + _ => return None, + }?; + } + output.push('"')?; + } + Some(()) +} + +fn write_text(output: &mut SvgOutput, text: &str) -> Option<()> { + for character in text.chars() { + match character { + '&' => output.push_str("&"), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + '\r' => output.push_str(" "), + character if is_xml_char(character) => output.push(character), + _ => return None, + }?; + } + Some(()) +} + +/// Writes a comment XML can hold: no `--` inside, no `-` at the end. +fn write_comment(output: &mut SvgOutput, text: &str) -> Option<()> { + if text.contains("--") || text.ends_with('-') || !text.chars().all(is_xml_char) { + return None; + } + output.push_str("")?; + Some(()) +} + +/// A character XML 1.0 allows in a document. +fn is_xml_char(character: char) -> bool { + matches!( + character, + '\t' | '\n' + | '\r' + | '\u{20}'..='\u{d7ff}' + | '\u{e000}'..='\u{fffd}' + | '\u{10000}'..='\u{10ffff}' + ) +} + +/// A name XML can write without a prefix. Conservative: a name it refuses +/// is never written, and anything this admits that XML reads otherwise +/// fails the equivalence check. +fn is_xml_local_name(name: &str) -> bool { + let mut characters = name.chars(); + characters + .next() + .is_some_and(|first| first.is_alphabetic() || first == '_') + && characters + .all(|character| character.is_alphanumeric() || matches!(character, '-' | '_' | '.')) +} + +#[cfg(test)] +mod tests { + use super::*; + + const LIMIT: usize = 1024 * 1024; + + fn signed(fragment: &str) -> Result, Reason> { + signable_svg(fragment, LIMIT) + } + + /// The serialization `fragment` is signed as, checked to be a standalone + /// SVG document whose tree, read as XML and again as HTML, is the + /// browser's tree of `fragment`. + fn normalized(fragment: &str) -> String { + let Ok(Cow::Owned(serialized)) = signed(fragment) else { + panic!("should serialize {fragment}, got {:?}", signed(fragment)); + }; + let browser = parse_html(fragment).expect("should parse the original as HTML"); + let browser = browser.root().expect("should have an svg root"); + let reparsed = parse_html(&serialized).expect("should parse the serialization as HTML"); + let reparsed = reparsed.root().expect("should keep one svg root"); + let document = parse_xml(&serialized).expect("should be standalone XML"); + assert_eq!( + document.root_element().tag_name().namespace(), + Some(SVG_NAMESPACE), + "should declare the SVG namespace" + ); + assert!( + equal(&Browser, browser, &Browser, reparsed), + "should read back as the browser's tree: {serialized}" + ); + assert!( + equal( + &Browser, + browser, + &Xml(PhantomData), + document.root_element() + ), + "should read as XML to the browser's tree: {serialized}" + ); + serialized + } + + fn assert_unchanged(fragment: &str) { + assert!( + matches!(signed(fragment), Ok(Cow::Borrowed(kept)) if kept == fragment), + "should sign {fragment} as written, got {:?}", + signed(fragment) + ); + } + + /// The first SVG element named `name` in `serialized`, read as XML. + fn element<'a, 'input>( + document: &'a roxmltree::Document<'input>, + name: &str, + ) -> roxmltree::Node<'a, 'input> { + document + .descendants() + .find(|node| node.has_tag_name((SVG_NAMESPACE, name))) + .unwrap_or_else(|| panic!("should hold a {name} element")) + } + + #[test] + fn the_first_of_two_case_variant_attributes_is_the_one_signed() { + // The captured logo: the browser keeps `viewbox`'s value as + // `viewBox` and drops the later `viewBox`. + let serialized = normalized( + "\n \n", + ); + let document = parse_xml(&serialized).expect("should be XML"); + let root = document.root_element(); + assert_eq!(root.attribute("viewBox"), Some("0 0 300 300.251")); + assert_eq!(root.attribute("viewbox"), None); + assert_eq!(root.attribute("version"), Some("1.1")); + assert_eq!(element(&document, "path").attribute("d"), Some("M0 0h1")); + } + + #[test] + fn an_svg_without_its_namespace_is_signed_in_it() { + let serialized = normalized( + "", + ); + let document = parse_xml(&serialized).expect("should be XML"); + let root = document.root_element(); + assert_eq!(root.attribute("fill"), Some("currentColor")); + assert_eq!(root.attribute("aria-hidden"), Some("true")); + assert_eq!( + element(&document, "path").attribute("d"), + Some("M18.244 2.25h3.308") + ); + } + + #[test] + fn a_standalone_svg_is_signed_as_written_whatever_its_declarations() { + assert_unchanged( + "", + ); + assert_unchanged( + "", + ); + assert_unchanged( + "", + ); + assert_unchanged( + "&’&", + ); + } + + #[test] + fn an_xlink_attribute_is_signed_in_the_xlink_namespace() { + for fragment in [ + "", + // A wrong binding is a declaration the browser ignores. + "", + ] { + let serialized = normalized(fragment); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!( + element(&document, "use").attribute((XLINK_NAMESPACE, "href")), + Some("#a"), + "for {fragment}" + ); + } + assert!( + !normalized("").contains("xmlns:xlink"), + "should declare XLink only where it is used" + ); + } + + #[test] + fn wrapped_attribute_whitespace_keeps_its_parsed_value() { + let serialized = normalized( + "", + ); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!( + element(&document, "path").attribute("d"), + Some("M0 0\n L1 1\tZ") + ); + } + + #[test] + fn html_character_references_are_signed_as_their_characters() { + let serialized = normalized( + "A &<B’", + ); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!(element(&document, "title").text(), Some("A\u{a0}&a�b"); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!(element(&document, "text").text(), Some("a\u{fffd}b")); + } + + #[test] + fn cdata_is_signed_as_its_text() { + let serialized = normalized(""); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!(element(&document, "style").text(), Some("a&e;"); + assert!(!serialized.contains("DOCTYPE"), "should carry no doctype"); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!(element(&document, "text").text(), Some("&e;")); + } + + #[test] + fn markup_that_breaks_out_of_the_svg_is_refused() { + for fragment in [ + "

x

", + "
x
", + "
x
", + "x", + ] { + assert_eq!( + signed(fragment), + Err(Reason::InlineSvgNotEquivalent), + "should refuse {fragment}" + ); + } + } + + #[test] + fn what_xml_cannot_carry_is_refused() { + for fragment in [ + "", + "", + "", + "￾", + "", + "", + ] { + assert_eq!( + signed(fragment), + Err(Reason::InlineSvgNotEquivalent), + "should refuse {fragment}" + ); + } + } + + #[test] + fn provenance_the_svg_carries_is_never_rewritten() { + for fragment in [ + // The writer's carrier, without and with the SVG namespace. + "QUJD", + "QUJD", + // An unclosed carrier element. + "QUJD", + // A C2PA namespace declared for anything. + "", + // An XMP provenance reference, which need not name C2PA. + "", + "https://example.com/m", + ] { + assert_eq!( + signed(fragment), + Err(Reason::SvgProvenanceNotEquivalent), + "should leave {fragment} as written" + ); + } + } + + #[test] + fn a_caption_that_mentions_c2pa_is_not_provenance() { + let serialized = normalized( + "Signed with C2PAc2pa", + ); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!(element(&document, "title").text(), Some("Signed with C2PA")); + } + + #[test] + fn serializer_checks_root_attributes_text_and_namespace_growth_before_writing() { + let fragment = ">&"; + let parsed = parse_html(fragment).expect("should parse"); + let root = parsed.root().expect("should have a root"); + let serialized = serialize(root, LIMIT).expect("should serialize"); + assert!(serialized.contains(SVG_DECLARATION)); + assert!(serialized.contains(XLINK_DECLARATION)); + for limit in 0..serialized.len() { + assert!( + serialize(root, limit).is_none(), + "must refuse limit {limit}" + ); + } + assert_eq!(serialize(root, serialized.len()), Some(serialized)); + + let fragment = format!("", "\"".repeat(1000)); + let parsed = parse_html(&fragment).expect("should parse a large root attribute"); + let Node::Element(element) = parsed.root().expect("should have a root").value() else { + panic!("should be an element"); + }; + let mut output = SvgOutput::new(127); + assert!(write_start(&mut output, element, &mut false).is_none()); + assert!(output.text.len() <= 127); + assert!(output.text.capacity() <= 127); + assert_eq!( + signable_svg(&fragment, 1100), + Err(Reason::InlineSvgNotEquivalent) + ); + + let fragment = format!("{}", ">".repeat(1000)); + assert_eq!( + signable_svg(&fragment, 1100), + Err(Reason::InlineSvgNotEquivalent) + ); + } + + #[test] + fn serializer_buffer_growth_is_bounded_amortized_and_not_the_media_limit() { + let mut output = SvgOutput::new(5_000_000); + assert_eq!(output.text.capacity(), 0); + output.push_str("", ">"), + ("\r", " "), + ] { + for limit in 0..escaped.len() { + let mut output = SvgOutput::new(limit); + assert!(write_text(&mut output, text).is_none()); + assert!(output.text.len() <= limit); + assert!(output.text.capacity() <= limit); + } + let mut output = SvgOutput::new(escaped.len()); + write_text(&mut output, text).expect("should fit the complete escape"); + assert_eq!(output.text, escaped); + } + for limit in 0..10 { + let mut output = SvgOutput::new(limit); + assert!(write_comment(&mut output, "art").is_none()); + assert!(output.text.len() <= limit); + assert!(output.text.capacity() <= limit); + } + let mut output = SvgOutput::new(10); + write_comment(&mut output, "art").expect("should fit the complete comment"); + assert_eq!(output.text, ""); + } + + #[test] + fn the_size_node_and_depth_bounds_refuse() { + let fragment = ""; + assert_eq!( + signable_svg(fragment, fragment.len() - 1), + Err(Reason::MediaTooLarge) + ); + let wide = format!( + "{}", + "".repeat(MAX_NODES) + ); + assert_eq!(signed(&wide), Err(Reason::InlineSvgNotEquivalent)); + let deep = format!( + "{}{}", + "".repeat(MAX_DEPTH), + "".repeat(MAX_DEPTH) + ); + assert_eq!(signed(&deep), Err(Reason::InlineSvgNotEquivalent)); + let nested = format!( + "{}{}", + "".repeat(MAX_DEPTH - 2), + "".repeat(MAX_DEPTH - 2) + ); + assert_unchanged(&nested); + } +} diff --git a/crates/provenance/encypher/src/tests.rs b/crates/provenance/encypher/src/tests.rs new file mode 100644 index 0000000000..bd2ddb588d --- /dev/null +++ b/crates/provenance/encypher/src/tests.rs @@ -0,0 +1,1717 @@ +use super::*; + +use std::sync::Arc; + +use base64::Engine as _; + +use edgezero_core::body::Body as EdgeBody; +use edgezero_core::key_value_store::KvPage; +use futures::executor::block_on; +use http::{Method, Request, header}; +use serde_json::json; +use trusted_server_core::auction::compile_auction_plan; +use trusted_server_core::auction::orchestrator::AuctionOrchestrator; +use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::ec::EcContext; +use trusted_server_core::html_processor::test_support::page_settings; +use trusted_server_core::integrations::IntegrationRegistry; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use trusted_server_core::platform::{ + KvError, PlatformHttpClient, PlatformKvStore, RuntimeServices, +}; +use trusted_server_core::publisher::{ + AppContext, AuctionDispatch, buffer_publisher_response_async, handle_publisher_request, + publisher_response_into_streaming_response, +}; + +const TOKEN: &str = "cwk_test_worker_token_value"; +const MARKER: &str = "\u{E0100}\u{E0101}\u{FE01}"; +const MANIFEST_URL: &str = "https://api.example.com/manifests/doc-1"; +const POLICY_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const ARTICLE: &str = "Harbor Council Votes\ +

Harbor council votes on the ferry schedule

\ +

The harbor council met on Tuesday evening and approved a revised ferry schedule \ + for the coming season.

Residents asked for earlier departures on weekdays.

\ +
Footer
"; + +fn module_settings(table: &str, serve: bool) -> String { + let entry = if serve { + format!("[[serve]]\nmedia_type = \"text/html\"\nmiddleware = [\"{MODULE}\"]\n") + } else { + String::new() + }; + format!("[provenance]\nmodule = \"encypher\"\n\n[provenance.encypher]\n{table}\n{entry}") +} + +fn table() -> String { + format!( + "api_base_url = \"https://api.example.com\"\ndomain = \"publisher.example.com\"\n\ + worker_token = \"{TOKEN}\"\n" + ) +} + +fn placed() -> Settings { + page_settings(&module_settings(&table(), true)) +} + +fn config_value() -> serde_json::Value { + json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": TOKEN, + }) +} + +fn parsed(value: &serde_json::Value) -> Result { + let config: EncypherConfig = + serde_json::from_value(value.clone()).map_err(|error| error.to_string())?; + config.validate().map_err(|error| error.to_string())?; + Ok(config) +} + +#[test] +fn module_constant_is_the_crate_folder() { + assert_eq!( + MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); +} + +#[test] +fn an_unselected_module_registers_nothing() { + let settings = page_settings(""); + + assert!(register(&settings).expect("should read").is_none()); + assert!(!validate(&settings).expect("should read")); +} + +#[test] +fn a_selected_and_placed_module_registers_its_middleware_and_media_signer() { + let settings = placed(); + + assert!(validate(&settings).expect("should validate")); + let registration = register(&settings) + .expect("should register") + .expect("a selected module with a worker token should register"); + let names: Vec<&str> = registration + .middleware + .iter() + .map(|middleware| middleware.middleware_id()) + .collect(); + assert_eq!( + names, + [MODULE, STYLESHEET_MIDDLEWARE], + "should supply the page and stylesheet middleware" + ); + assert_eq!( + registration + .media_signer + .as_ref() + .map(|signer| signer.signer_id()), + Some(MODULE), + "should register the media signer alongside the text path" + ); + let stylesheet = ®istration.middleware[1]; + assert!(stylesheet.handles_media_type("text/css")); + assert!( + stylesheet.handles_path("/assets/site.css"), + "the stylesheet path should not inherit the page path's asset exclusions" + ); +} + +#[test] +fn a_module_named_in_no_serve_entry_is_refused() { + let settings = page_settings(&module_settings(&table(), false)); + + let error = validate(&settings).expect_err("should refuse the document"); + + assert!( + format!("{error:?}").contains("no [[serve]] entry names"), + "should name the missing entry: {error:?}" + ); +} + +#[test] +fn a_blank_worker_token_registers_nothing() { + for token in ["", " \t"] { + let table = format!( + "api_base_url = \"https://api.example.com\"\ndomain = \"publisher.example.com\"\n\ + worker_token = \"{token}\"\n" + ); + let settings = page_settings(&module_settings(&table, true)); + + assert!( + register(&settings).expect("should read").is_none(), + "token {token:?} should not register" + ); + } +} + +#[test] +fn required_settings_and_bounds_are_enforced() { + for field in ["domain", "api_base_url", "worker_token"] { + let mut config = config_value(); + config + .as_object_mut() + .expect("should be an object") + .remove(field); + assert!( + parsed(&config).is_err(), + "config without {field} should fail" + ); + } + for (field, value) in [ + ("api_base_url", json!("http://api.example.com")), + ("api_base_url", json!("https://api.example.com/?q=1")), + ("domain", json!("Publisher.example.com")), + ("domain", json!("www.publisher.example.com")), + ("domain", json!("publisher.example.com/path")), + ("domain", json!("localhost")), + ("timeout_ms", json!(100)), + ("timeout_ms", json!(20_000)), + ("min_text_length", json!(10)), + ("negative_ttl_seconds", json!(10)), + ("policy_ttl_seconds", json!(301)), + ("article_selector", json!("a".repeat(92))), + ("max_media_bytes", json!(100)), + ("max_media_bytes", json!(8_000_001)), + ("max_media_sign_calls_per_document", json!(0)), + ("max_media_sign_calls_per_document", json!(129)), + ("max_large_media_bytes", json!(1)), + ( + "max_large_media_bytes", + json!(MAX_PREPARED_SOURCE_BYTES + 1), + ), + ("max_large_media_ms", json!(4_999)), + ("max_large_media_ms", json!(600_001)), + ("enabled", json!(true)), + ("path_prefixes", json!(["/news/"])), + ] { + let mut config = config_value(); + config[field] = value.clone(); + assert!( + parsed(&config).is_err(), + "{field} = {value} should be rejected" + ); + } + for value in [0, MIN_LARGE_MEDIA_BYTES, MAX_PREPARED_SOURCE_BYTES] { + let mut config = config_value(); + config["max_large_media_bytes"] = json!(value); + assert!( + parsed(&config).is_ok(), + "max_large_media_bytes = {value} should be accepted, 0 turning the lane off" + ); + } + for limit in [1, 64, 128] { + let mut config = config_value(); + config["max_media_sign_calls_per_document"] = json!(limit); + let accepted = parsed(&config).expect("bounded media-call limit"); + assert_eq!(accepted.max_media_sign_calls_per_document, limit); + } + for (max_hold_ms, timeout_ms, valid) in [ + (499, 250, false), + (500, 250, true), + (15_000, 3_000, true), + (15_001, 3_000, false), + (3_000, 3_000, false), + (3_001, 3_000, true), + ] { + let mut config = config_value(); + config["max_hold_ms"] = json!(max_hold_ms); + config["timeout_ms"] = json!(timeout_ms); + assert_eq!( + parsed(&config).is_ok(), + valid, + "max_hold_ms {max_hold_ms} with timeout_ms {timeout_ms}" + ); + } +} + +#[test] +fn article_selector_rejects_malformed_classes_at_settings_admission() { + for selector in [ + ".", + "..article", + ".123article", + ".article.body", + ".article body", + ".article>p", + ".article,main", + ".article#body", + ".article:hover", + ".article[data-body=yes]", + ".article\\:body", + ] { + let table = format!("{}article_selector = {}\n", table(), json!(selector)); + let settings = page_settings(&module_settings(&table, true)); + assert!( + validate(&settings).is_err(), + "malformed class selector {selector:?} should fail settings admission" + ); + assert!( + register(&settings).is_err(), + "malformed class selector {selector:?} should not register" + ); + } +} + +#[test] +fn article_selector_rejects_malformed_attributes_at_settings_admission() { + for selector in [ + "[", + "[itemprop]", + "[=articleBody]", + "[itemprop=]", + "[itemprop=articleBody", + "[itemprop=articleBody]p", + "[itemprop~=articleBody]", + "[itemprop=articleBody main]", + "[itemprop=articleBody=main]", + "[itemprop=[articleBody]]", + "[itemprop=\"articleBody']", + "[itemprop='articleBody' i]", + "[itemprop=articleBody][role=main]", + "[itemprop=articleBody],main", + "[itemprop='article\\Body']", + ] { + let table = format!("{}article_selector = {}\n", table(), json!(selector)); + let settings = page_settings(&module_settings(&table, true)); + assert!( + validate(&settings).is_err(), + "malformed attribute selector {selector:?} should fail settings admission" + ); + assert!( + register(&settings).is_err(), + "malformed attribute selector {selector:?} should not register" + ); + } +} + +#[test] +fn article_selector_rejects_malformed_tags_at_settings_admission() { + for selector in [ + " ", + " article", + "article ", + "123article", + "
", + "article.body", + "article p", + "article>p", + "article+p", + "article~p", + "article,main", + "article#body", + "article:hover", + "article[role=main]", + "#article", + "*", + ] { + let table = format!("{}article_selector = {}\n", table(), json!(selector)); + let settings = page_settings(&module_settings(&table, true)); + assert!( + validate(&settings).is_err(), + "malformed tag selector {selector:?} should fail settings admission" + ); + assert!( + register(&settings).is_err(), + "malformed tag selector {selector:?} should not register" + ); + } +} + +#[test] +fn article_selector_accepts_supported_syntax_and_selects_the_override() { + let selected = "Selected."; + let html = format!("
Fallback.
{selected}"); + for selector in [ + ".story-body", + "._story", + ".-story", + ".--story", + "[itemprop=articleBody]", + "[itemprop='articleBody']", + "[itemprop=\"articleBody\"]", + "[data-body='article body']", + "custom-story", + "CUSTOM-STORY", + ] { + let table = format!("{}article_selector = {}\n", table(), json!(selector)); + let settings = page_settings(&module_settings(&table, true)); + assert!( + validate(&settings).expect("should admit supported selector"), + "supported selector {selector:?} should select the module" + ); + assert!( + register(&settings) + .expect("should register supported selector") + .is_some(), + "supported selector {selector:?} should register" + ); + let boundary = boundary::find_article_boundary(&html, Some(selector)) + .expect("should find the configured override"); + assert_eq!( + boundary.selector, + format!("override:{selector}"), + "supported selector should take priority over the article fallback" + ); + assert_eq!(html.get(boundary.start..boundary.end), Some(selected)); + } +} + +#[test] +fn article_selector_empty_or_valid_but_absent_keeps_article_fallback() { + let selected = "

Fallback article.

"; + let html = format!("
Other region.
{selected}"); + for selector in ["", ".absent-story", "[data-body=absent]", "custom-story"] { + let table = format!("{}article_selector = {}\n", table(), json!(selector)); + let settings = page_settings(&module_settings(&table, true)); + assert!( + validate(&settings).expect("should admit empty or absent selector"), + "selector {selector:?} should select the module" + ); + let boundary = boundary::find_article_boundary(&html, Some(selector)) + .expect("should find the fallback article"); + assert_eq!(boundary.selector, "article"); + assert_eq!(html.get(boundary.start..boundary.end), Some(selected)); + } +} + +#[test] +fn excluded_paths_leave_the_middleware_out_of_the_chain() { + let middleware = Encypher { + signer: Arc::new(Signer::new( + parsed(&config_value()).expect("should parse"), + 5_000_000, + )), + }; + + assert!(middleware.handles_path("/news/harbor-vote")); + for path in [ + "/wp-admin/post", + "/feed/", + "/sitemap.xml", + "/app.js", + "/api/v1/x", + ] { + assert!(!middleware.handles_path(path), "{path} should be left out"); + } + assert_eq!(middleware.phases(), [MiddlewarePhase::Serve]); +} + +/// A runtime KV store held in memory. +#[derive(Default)] +struct MemoryKv(std::sync::Mutex>); + +#[async_trait::async_trait(?Send)] +impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + Ok(self.0.lock().expect("should lock").get(key).cloned()) + } + + async fn put_bytes(&self, key: &str, value: bytes::Bytes) -> Result<(), KvError> { + self.0 + .lock() + .expect("should lock") + .insert(key.to_owned(), value); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: bytes::Bytes, + _ttl: std::time::Duration, + ) -> Result<(), KvError> { + self.put_bytes(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.0.lock().expect("should lock").remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(KvPage::default()) + } +} + +/// An origin answering [`ARTICLE`] with `cache_control`, then the policy and +/// sign answers of the API. +fn stub(cache_control: &str) -> Arc { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + ARTICLE.as_bytes().to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", cache_control), + ], + ); + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + stub.push_response( + 200, + serde_json::to_vec(&json!({ + "success": true, + "policy_digest": POLICY_DIGEST, + "embedding_plan": { + "index_unit": "codepoint", + "operations": [{ "insert_after_index": 5, "marker": MARKER }], + }, + "manifest_url": MANIFEST_URL, + "verification_url": "https://api.example.com/verify/doc-1", + "document_id": "doc-1", + })) + .expect("should encode"), + ); + stub +} + +fn origin_stub(cache_control: &str) -> Arc { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + ARTICLE.as_bytes().to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", cache_control), + ], + ); + stub +} + +fn page_request(method: Method) -> Request { + Request::builder() + .method(method) + .uri("https://publisher.example.com/news/harbor-vote?utm_source=feed") + .header(header::HOST, "publisher.example.com") + .body(EdgeBody::empty()) + .expect("should build the page request") +} + +/// The response a reader receives from the publisher path, through the +/// buffered or the streaming finalizer. +fn served_response( + settings: &Settings, + services: &RuntimeServices, + request: Request, + streaming: bool, +) -> http::Response { + try_served_response(settings, services, request, streaming).expect("should finish the page") +} + +fn try_served_response( + settings: &Settings, + services: &RuntimeServices, + request: Request, + streaming: bool, +) -> Result, error_stack::Report> { + let registry = IntegrationRegistry::with_registrations(settings, &[builder()]) + .expect("should build the registry"); + // As every adapter does, the request's services carry the registry's + // media signer. + let services = &services.clone().with_media_signer(registry.media_signer()); + let plan = compile_auction_plan(settings).expect("should compile the auction plan"); + let orchestrator = Arc::new(AuctionOrchestrator::from_plan(Arc::new(plan), None)); + let method = request.method().clone(); + block_on(async { + let mut ec_context = + EcContext::read_from_request_resolving_geo(settings, &request, services) + .await + .expect("should read the request's state"); + let publisher = handle_publisher_request( + AppContext { + settings, + integration_registry: ®istry, + }, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SurrogateControl, + ) + .await?; + if streaming { + publisher_response_into_streaming_response( + publisher, + &method, + Arc::new(settings.clone()), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await + } else { + buffer_publisher_response_async( + publisher, + &method, + settings, + ®istry, + &orchestrator, + services, + ) + .await + } + }) +} + +/// The page body a reader receives from the publisher path. +fn served( + settings: &Settings, + services: &RuntimeServices, + request: Request, + streaming: bool, +) -> String { + let response = served_response(settings, services, request, streaming); + block_on(async { + let body = response + .into_body() + .into_bytes_bounded(16 * 1024 * 1024) + .await + .expect("should read the body"); + String::from_utf8(body.to_vec()).expect("should be UTF-8") + }) +} + +fn services_for(stub: &Arc) -> RuntimeServices { + services_for_kv(stub, Arc::new(MemoryKv::default())) +} + +fn services_for_kv(stub: &Arc, kv: Arc) -> RuntimeServices { + build_services_with_http_client(Arc::clone(stub) as Arc) + .with_kv_store(kv) +} + +#[test] +fn the_publisher_path_signs_an_article_through_both_finalizers() { + for streaming in [false, true] { + let stub = stub("public, max-age=60"); + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + streaming, + ); + + assert!( + page.contains(MARKER), + "streaming={streaming}: should embed the marker: {page}" + ); + assert!( + page.contains("c2pa-manifest-url") && page.contains(MANIFEST_URL), + "streaming={streaming}: should write the manifest meta tag" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET", "GET", "POST"], + "streaming={streaming}: origin, policy, then sign" + ); + // The test request carries no TLS signal, so the scheme the publisher + // path detects is `http`. + let policy_uri = &stub.recorded_request_uris()[1]; + assert!( + policy_uri + .ends_with("page_url=http%3A%2F%2Fpublisher.example.com%2Fnews%2Fharbor-vote"), + "should send the page address without its query: {policy_uri}" + ); + } +} + +#[test] +fn an_article_on_a_publisher_addressed_by_path_is_signed_for_its_own_address() { + let stub = stub("public, max-age=60"); + let mut request = page_request(Method::GET); + request + .extensions_mut() + .insert(trusted_server_core::addressing::RemovedPathPrefix::new( + "/www.site.example".to_owned(), + )); + + let page = served(&placed(), &services_for(&stub), request, false); + + assert!(page.contains(MARKER), "should sign the article: {page}"); + let policy_uri = &stub.recorded_request_uris()[1]; + assert!( + policy_uri.ends_with( + "page_url=http%3A%2F%2Fpublisher.example.com%2Fwww.site.example%2Fnews%2Fharbor-vote" + ), + "should ask the policy for the address the reader asked for: {policy_uri}" + ); + let sign = String::from_utf8(stub.recorded_request_bodies()[2].clone()).expect("JSON body"); + assert!( + sign.contains( + "\"page_url\":\"http://publisher.example.com/www.site.example/news/harbor-vote\"" + ), + "should sign the article for the address the reader asked for: {sign}" + ); +} + +/// The unsigned bytes produced by the same public publisher pipeline before +/// its final writer applies the configured bound. Core middleware may rewrite +/// the origin body before Encypher sees it, so the origin fixture length is not +/// the final unsigned length. +fn unsigned_public_page() -> String { + let stub = origin_stub("public, max-age=60"); + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + false, + ); + assert!( + page.len() > ARTICLE.len(), + "fixture must retain the HTML rewriter's output overhead" + ); + page +} + +#[test] +fn fresh_expansion_over_the_buffered_limit_returns_the_unsigned_page() { + let unsigned = unsigned_public_page(); + let mut settings = placed(); + settings.publisher.max_buffered_body_bytes = unsigned.len(); + let stub = stub("public, max-age=60"); + let response = served_response( + &settings, + &services_for(&stub), + page_request(Method::GET), + false, + ); + + assert_eq!(response.status(), 200); + let body = block_on(response.into_body().into_bytes_bounded(unsigned.len())) + .expect("the unsigned pipeline output should fit its exact buffered limit"); + assert_eq!(body.as_ref(), unsigned.as_bytes()); + assert_eq!(stub.recorded_request_methods(), ["GET", "GET", "POST"]); +} + +#[test] +fn cached_expansion_over_the_buffered_limit_returns_the_unsigned_page() { + let unsigned = unsigned_public_page(); + let kv = Arc::new(MemoryKv::default()); + let first = stub("public, max-age=60"); + let signed = served( + &placed(), + &services_for_kv(&first, Arc::clone(&kv)), + page_request(Method::GET), + false, + ); + assert!(signed.contains(MARKER), "should seed a valid plan"); + + let mut settings = placed(); + settings.publisher.max_buffered_body_bytes = unsigned.len(); + let cached = origin_stub("public, max-age=60"); + let response = served_response( + &settings, + &services_for_kv(&cached, kv), + page_request(Method::GET), + false, + ); + + assert_eq!(response.status(), 200); + let body = block_on(response.into_body().into_bytes_bounded(unsigned.len())) + .expect("the unsigned pipeline output should fit its exact buffered limit"); + assert_eq!(body.as_ref(), unsigned.as_bytes()); + assert_eq!( + cached.recorded_request_methods(), + ["GET"], + "the cached policy and plan should make no Encypher API request" + ); +} + +#[test] +fn a_page_the_origin_marked_private_is_never_sent() { + let stub = stub("private, max-age=0"); + + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + false, + ); + + assert!( + !page.contains(MARKER), + "should not sign a reader's own page" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET"], + "should call the origin and nothing else" + ); +} + +/// The pixels of a small image an origin embeds or serves. +const SOURCE_IMAGE: &[u8] = b"\x89PNG\r\n\x1a\n-source-pixels"; +/// The same image with an embedded C2PA manifest, as the media endpoint +/// answers it. +const SIGNED_IMAGE: &[u8] = b"\x89PNG\r\n\x1a\n-source-pixels-caBX-jumb-c2pa-manifest"; + +fn encoded(bytes: &[u8]) -> String { + base64::engine::general_purpose::STANDARD.encode(bytes) +} + +/// The headers of an embedded-manifest answer of the media endpoint for +/// `body` of `content_type`. +fn media_answer_as(body: &[u8], content_type: &str) -> Vec<(String, String)> { + [ + ("content-type", content_type.to_owned()), + ("x-encypher-policy-digest", POLICY_DIGEST.to_owned()), + ("x-encypher-signed-sha256", media_api::sha256_hex(body)), + ("x-encypher-manifest-instance-id", "xmp:iid:0001".to_owned()), + ("x-encypher-manifest-disposition", "embedded".to_owned()), + ("x-encypher-manifest-link-required", "false".to_owned()), + ( + "x-encypher-verification-url", + "https://verify.example.com/doc-2".to_owned(), + ), + ("x-encypher-document-id", "doc-2".to_owned()), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect() +} + +/// [`ARTICLE`] with a lazily loaded embedded image inside the article. +fn article_with_image() -> String { + ARTICLE.replace( + "
", + &format!( + "\"Harbor\"
", + encoded(SOURCE_IMAGE) + ), + ) +} + +/// An origin answering `body` as `content_type`, publicly cacheable. +fn origin_answer(stub: &StubHttpClient, body: &[u8], content_type: &str) { + stub.push_response_with_headers( + 200, + body.to_vec(), + vec![ + ("content-type", content_type), + ("cache-control", "public, max-age=60"), + ], + ); +} + +/// The policy answer the media lane asks for live, before it signs or reuses +/// embedded media. +fn live_policy(stub: &StubHttpClient) { + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); +} + +#[test] +fn a_revoked_credential_stops_embedded_media_reuse_while_the_cached_policy_lives() { + let kv = Arc::new(MemoryKv::default()); + let first = Arc::new(StubHttpClient::new()); + origin_answer( + &first, + article_with_image().as_bytes(), + "text/html; charset=utf-8", + ); + text_answers(&first); + live_policy(&first); + first.push_response_with_headers(200, SIGNED_IMAGE.to_vec(), media_answer(SIGNED_IMAGE)); + let warmed = served( + &placed(), + &services_for_kv(&first, Arc::clone(&kv)), + page_request(Method::GET), + false, + ); + assert!( + warmed.contains(&encoded(SIGNED_IMAGE)), + "should warm the policy and media caches" + ); + + // The cached policy and the cached signed image are both still fresh; + // the service now refuses the credential. + let revoked = Arc::new(StubHttpClient::new()); + origin_answer( + &revoked, + article_with_image().as_bytes(), + "text/html; charset=utf-8", + ); + revoked.push_response(403, b"{}".to_vec()); + let page = served( + &placed(), + &services_for_kv(&revoked, kv), + page_request(Method::GET), + false, + ); + + assert!( + !page.contains(&encoded(SIGNED_IMAGE)), + "should never reuse a signed image under a revoked credential" + ); + assert!( + page.contains(&encoded(SOURCE_IMAGE)), + "should deliver the image as the origin wrote it" + ); + assert!( + !revoked + .recorded_request_methods() + .iter() + .any(|method| method == "POST"), + "should sign nothing" + ); +} + +/// The policy and article-sign answers [`stub`] queues after the origin. +fn text_answers(stub: &StubHttpClient) { + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + stub.push_response( + 200, + serde_json::to_vec(&json!({ + "success": true, + "policy_digest": POLICY_DIGEST, + "embedding_plan": { + "index_unit": "codepoint", + "operations": [{ "insert_after_index": 5, "marker": MARKER }], + }, + "manifest_url": MANIFEST_URL, + })) + .expect("should encode"), + ); +} + +#[test] +fn the_publisher_path_signs_the_article_then_its_embedded_image() { + for streaming in [false, true] { + let stub = Arc::new(StubHttpClient::new()); + origin_answer( + &stub, + article_with_image().as_bytes(), + "text/html; charset=utf-8", + ); + text_answers(&stub); + live_policy(&stub); + stub.push_response_with_headers(200, SIGNED_IMAGE.to_vec(), media_answer(SIGNED_IMAGE)); + + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + streaming, + ); + + assert!( + page.contains(MARKER), + "streaming={streaming}: should sign the text" + ); + assert!( + page.contains(&format!( + "src=\"data:image/png;base64,{}\" alt=\"Harbor\"", + encoded(SIGNED_IMAGE) + )), + "streaming={streaming}: should deliver the signed image bytes in place: {page}" + ); + assert!( + !page.contains(&encoded(SOURCE_IMAGE)), + "streaming={streaming}: should leave no unsigned copy" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET", "GET", "POST", "GET", "POST"], + "streaming={streaming}: origin, policy, text sign, live policy, media sign" + ); + assert!( + stub.recorded_request_uris()[4].ends_with("/api/v1/public/cdn/media/sign"), + "streaming={streaming}: should call the media endpoint" + ); + let media_body = &stub.recorded_request_bodies()[4]; + assert!( + media_body + .windows(SOURCE_IMAGE.len()) + .any(|window| window == SOURCE_IMAGE), + "streaming={streaming}: should send the decoded source bytes" + ); + } +} + +#[test] +fn a_refused_embedded_image_stays_as_written_and_the_text_is_still_signed() { + let stub = Arc::new(StubHttpClient::new()); + origin_answer( + &stub, + article_with_image().as_bytes(), + "text/html; charset=utf-8", + ); + text_answers(&stub); + live_policy(&stub); + stub.push_response( + 422, + serde_json::to_vec(&json!({ + "success": false, + "error": {"code": "UNSUPPORTED_MEDIA_TYPE", "message": "x"}, + })) + .expect("should encode"), + ); + + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + false, + ); + + assert!(page.contains(MARKER), "should still sign the text"); + assert!( + page.contains(&encoded(SOURCE_IMAGE)), + "should leave the unsigned image as the origin wrote it" + ); + assert!( + !page.contains(&encoded(SIGNED_IMAGE)), + "should never present unsigned bytes as signed" + ); +} + +/// The module's settings with a stylesheet entry, scoped by `path` when one +/// is given. +fn stylesheet_settings(path: Option<&str>) -> String { + let path = path.map_or(String::new(), |path| format!("path = \"{path}\"\n")); + format!( + "{}\n[[serve]]\nmedia_type = \"text/css\"\n{path}middleware = [\"{STYLESHEET_MIDDLEWARE}\"]\n", + module_settings(&table(), true) + ) +} + +#[test] +fn a_stylesheet_entry_for_every_path_is_refused() { + for path in [None, Some("/")] { + let settings = page_settings(&stylesheet_settings(path)); + let error = validate(&settings).expect_err("should refuse an unscoped entry"); + assert!( + format!("{error:?}").contains("integrity"), + "{path:?}: should explain the integrity risk: {error:?}" + ); + } + let scoped = page_settings(&stylesheet_settings(Some("/assets/"))); + assert!(validate(&scoped).expect("should accept a scoped entry")); +} + +#[test] +fn the_publisher_path_signs_media_a_stylesheet_embeds() { + let settings = page_settings(&stylesheet_settings(Some("/assets/"))); + let stylesheet = format!( + ".hero{{background-image:url(\"data:image/png;base64,{}\")}}", + encoded(SOURCE_IMAGE) + ); + let stub = Arc::new(StubHttpClient::new()); + origin_answer(&stub, stylesheet.as_bytes(), "text/css; charset=utf-8"); + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + live_policy(&stub); + stub.push_response_with_headers(200, SIGNED_IMAGE.to_vec(), media_answer(SIGNED_IMAGE)); + let request = Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/assets/site.css") + .header(header::HOST, "publisher.example.com") + .body(EdgeBody::empty()) + .expect("should build the stylesheet request"); + + let served_stylesheet = served(&settings, &services_for(&stub), request, false); + + assert_eq!( + served_stylesheet, + format!( + ".hero{{background-image:url(\"data:image/png;base64,{}\")}}", + encoded(SIGNED_IMAGE) + ), + "should replace the embedded image with its signed bytes and nothing else" + ); +} + +/// The headers of an embedded-manifest PNG answer for `body`. +fn media_answer(body: &[u8]) -> Vec<(String, String)> { + media_answer_as(body, "image/png") +} + +fn media_request(path: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("https://publisher.example.com{path}")) + .header(header::HOST, "publisher.example.com") + .header(header::ACCEPT, "*/*") + .body(EdgeBody::empty()) + .expect("should build the media request") +} + +fn body_of(response: http::Response) -> bytes::Bytes { + block_on(response.into_body().into_bytes_bounded(16 * 1024 * 1024)) + .expect("should read the body") +} + +#[test] +fn the_publisher_path_delivers_the_signed_bytes_of_images_audio_and_video_on_any_path() { + for (path, content_type) in [ + ("/images/harbor.png", "image/png"), + ("/media/4711", "image/png"), + ("/video/ferry.mp4", "video/mp4"), + ("/audio/council.mp3", "audio/mpeg"), + ] { + let stub = Arc::new(StubHttpClient::new()); + origin_answer(&stub, SOURCE_IMAGE, content_type); + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + stub.push_response_with_headers( + 200, + SIGNED_IMAGE.to_vec(), + media_answer_as(SIGNED_IMAGE, content_type), + ); + + let response = served_response(&placed(), &services_for(&stub), media_request(path), false); + + assert_eq!(response.status(), 200, "{path}: should answer"); + let headers = response.headers().clone(); + assert_eq!( + headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some(content_type), + "{path}: should keep the media type" + ); + assert_eq!( + headers + .get("x-encypher-document-id") + .and_then(|value| value.to_str().ok()), + Some("doc-2"), + "{path}: should name the signed document" + ); + assert_eq!( + headers + .get(header::ETAG) + .and_then(|value| value.to_str().ok()), + Some(format!("\"sig-{}\"", media_api::sha256_hex(SIGNED_IMAGE)).as_str()), + "{path}: should validate the signed bytes, not the origin's" + ); + assert_eq!( + body_of(response).as_ref(), + SIGNED_IMAGE, + "{path}: should deliver the exact signed bytes" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET", "GET", "POST"], + "{path}: origin, policy, media sign" + ); + let metadata = &stub.recorded_request_bodies()[2]; + assert!( + metadata + .windows(SOURCE_IMAGE.len()) + .any(|window| window == SOURCE_IMAGE), + "{path}: should send the captured origin bytes" + ); + assert!( + metadata + .windows(b"\"origin_host\":\"".len()) + .any(|window| window == b"\"origin_host\":\""), + "{path}: should name the host core fetched from" + ); + } +} + +#[test] +fn a_failed_media_signature_delivers_the_origin_bytes_unmarked() { + let stub = Arc::new(StubHttpClient::new()); + origin_answer(&stub, SOURCE_IMAGE, "image/png"); + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + // An answer whose bytes are not the ones it names is never delivered. + stub.push_response_with_headers( + 200, + b"\x89PNG\r\n\x1a\nswapped-c2pa".to_vec(), + media_answer(SIGNED_IMAGE), + ); + + let response = served_response( + &placed(), + &services_for(&stub), + media_request("/images/harbor.png"), + false, + ); + + assert!( + response.headers().get("x-encypher-document-id").is_none(), + "should claim no signature" + ); + assert_eq!( + body_of(response).as_ref(), + SOURCE_IMAGE, + "should deliver the origin's bytes as they were" + ); +} + +#[test] +fn a_media_segment_passes_through_as_the_origin_sent_it() { + // A C2PA verifiable-segment-info segment is bound by its stream's init + // manifest. Signing it alone would fail, and the unsigned fallback would + // make it private and drop its validators. + const SEGMENT: &[u8] = b"\0\0\0\x18stypmsdh\0\0\0\0msdhmsixmoof-mdat-emsg"; + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + SEGMENT.to_vec(), + vec![ + ("content-type", "video/iso.segment"), + ("cache-control", "public, max-age=60"), + ("etag", "\"segment-7\""), + ], + ); + + let response = served_response( + &placed(), + &services_for(&stub), + media_request("/live/ferry/segment-7.m4s"), + false, + ); + + assert_eq!(response.status(), 200, "should answer"); + let headers = response.headers().clone(); + assert_eq!( + headers + .get(header::ETAG) + .and_then(|value| value.to_str().ok()), + Some("\"segment-7\""), + "should keep the origin's validator" + ); + assert!( + headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| !value.contains("private")), + "should not make the segment private" + ); + assert!( + headers.get("x-encypher-document-id").is_none(), + "should claim no signature" + ); + assert_eq!( + body_of(response).as_ref(), + SEGMENT, + "should deliver the segment byte for byte" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET"], + "should ask neither the policy nor the signing endpoint" + ); +} + +/// One ISO BMFF box of `kind` around `body`. +fn bmff_box(kind: &[u8; 4], body: &[u8]) -> Vec { + let size = u32::try_from(body.len() + 8).expect("should fit a 32-bit size"); + [&size.to_be_bytes()[..], kind, body].concat() +} + +#[test] +fn fragmented_stream_parts_typed_as_mp4_pass_through_as_the_origin_sent_them() { + let init = [ + bmff_box(b"ftyp", b"cmfc\0\0\0\0iso6cmfc"), + bmff_box( + b"moov", + &[ + bmff_box(b"mvhd", &[0; 100]), + bmff_box(b"mvex", &bmff_box(b"trex", &[0; 24])), + bmff_box(b"pssh", &[0; 32]), + ] + .concat(), + ), + ] + .concat(); + let segment = [ + bmff_box(b"styp", b"msdh\0\0\0\0msdhmsix"), + bmff_box(b"emsg", b"\0\0\0\0urn:c2pa:verifiable-segment-info\0fseg\0"), + bmff_box(b"sidx", &[0; 32]), + bmff_box(b"moof", &bmff_box(b"mfhd", &[0; 8])), + bmff_box(b"mdat", b"coded-samples"), + ] + .concat(); + for (path, part) in [ + ("/live/ferry/init.mp4", init), + ("/live/ferry/7.m4s", segment), + ] { + // No `no-transform`: the bytes alone keep the part from being signed. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + part.clone(), + vec![ + ("content-type", "video/mp4"), + ("cache-control", "public, max-age=60"), + ("etag", "\"part-7\""), + ], + ); + + let response = served_response(&placed(), &services_for(&stub), media_request(path), false); + + assert_eq!(response.status(), 200, "{path}: should answer"); + let headers = response.headers().clone(); + assert_eq!( + headers + .get(header::ETAG) + .and_then(|value| value.to_str().ok()), + Some("\"part-7\""), + "{path}: should keep the origin's validator" + ); + assert!( + headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| !value.contains("private")), + "{path}: should not make the part private" + ); + assert!( + headers.get(header::LINK).is_none() && headers.get("x-encypher-document-id").is_none(), + "{path}: should add no manifest" + ); + assert_eq!( + body_of(response).as_ref(), + part.as_slice(), + "{path}: should deliver the part byte for byte" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET"], + "{path}: should ask neither the policy nor the signing endpoint" + ); + } +} + +#[test] +fn a_progressive_mp4_is_still_handed_to_the_signing_endpoint() { + let progressive = [ + bmff_box(b"ftyp", b"isom\0\0\0\0isomavc1"), + bmff_box(b"moov", &bmff_box(b"trak", &[0; 16])), + bmff_box(b"mdat", b"coded-samples"), + ] + .concat(); + let stub = Arc::new(StubHttpClient::new()); + origin_answer(&stub, &progressive, "video/mp4"); + live_policy(&stub); + stub.push_response_with_headers( + 200, + SIGNED_IMAGE.to_vec(), + media_answer_as(SIGNED_IMAGE, "video/mp4"), + ); + + let _response = served_response( + &placed(), + &services_for(&stub), + media_request("/video/ferry.mp4"), + false, + ); + + assert_eq!( + stub.recorded_request_methods(), + ["GET", "GET", "POST"], + "origin, policy, media sign" + ); + let sent = &stub.recorded_request_bodies()[2]; + assert!( + sent.windows(progressive.len()) + .any(|window| window == progressive.as_slice()), + "should send the progressive file's exact bytes" + ); +} + +/// Signed source bytes deliberately contain targets of core's attribute and +/// text URL replacement before the provenance carrier/reference appears. +fn protected_svg(reference: bool) -> String { + let provenance = if reference { + "" + } else { + "QUJD" + }; + format!( + "\ + \ + https://origin.example.com/\ + \ + \ + {}{provenance}", + encoded(SOURCE_IMAGE), + "origin.example.com ".repeat(1024), + ) +} + +fn article_with_protected_svg(svg: &str) -> String { + ARTICLE.replace( + "
", + &format!( + "{svg}\ + https://origin.example.com/outside" + ), + ) +} + +fn assert_source_and_surroundings(page: &str, svg: &str) { + assert!( + page.contains(svg), + "must retain the complete original SVG byte slice" + ); + assert!( + page.contains("http://publisher.example.com/outside"), + "core's ordinary outside URL edits must still run: {page}" + ); + assert!(page.contains(MARKER), "article signing must still run"); + assert!( + !page.contains("` and `` are whole comments, and `--!>` also ends one. +pub(crate) fn comment_end(bytes: &[u8], start: usize) -> Option { + let body = start + 4; + if starts_with_at(bytes, body, b">") { + return Some(body + 1); + } + if starts_with_at(bytes, body, b"->") { + return Some(body + 2); + } + let dash = find_bytes(bytes, b"-->", body).map(|end| end + 3); + let bang = find_bytes(bytes, b"--!>", body).map(|end| end + 4); + match (dash, bang) { + (Some(dash), Some(bang)) => Some(dash.min(bang)), + (end, None) | (None, end) => end, + } +} + +/// Tokenizer states between a tag's name and its closing `>`. +#[derive(Clone, Copy)] +enum TagState { + Name, + BeforeAttributeName, + AttributeName, + BeforeValue, + Quoted(u8), + Unquoted, +} + +/// Returns the offset of the `>` ending the tag whose name starts at `start`. +/// As in the HTML tokenizer, a quote delimits only a value it begins. +pub(crate) fn tag_end(bytes: &[u8], start: usize) -> Option { + let mut state = TagState::Name; + for (offset, byte) in bytes.get(start..)?.iter().copied().enumerate() { + let space = byte.is_ascii_whitespace(); + state = match state { + TagState::Quoted(quote) => { + if byte == quote { + TagState::BeforeAttributeName + } else { + state + } + } + _ if byte == b'>' => return Some(start + offset), + TagState::Name if space || byte == b'/' => TagState::BeforeAttributeName, + TagState::BeforeAttributeName if !space && byte != b'/' => TagState::AttributeName, + TagState::AttributeName if byte == b'=' => TagState::BeforeValue, + TagState::AttributeName if byte == b'/' => TagState::BeforeAttributeName, + TagState::BeforeValue if byte == b'"' || byte == b'\'' => TagState::Quoted(byte), + TagState::BeforeValue if !space => TagState::Unquoted, + TagState::Unquoted if space => TagState::BeforeAttributeName, + _ => state, + }; + } + None +} + +fn match_literal_suffix(units: &I, expected: &str) -> Option<(I, usize)> +where + I: Clone + Iterator, +{ + let mut trial = units.clone(); + let mut end = None; + for expected_char in expected.chars() { + let item = trial.next()?; + if item.ch != expected_char { + return None; + } + end = Some(item.end); + } + Some((trial, end?)) +} + +fn match_numeric_suffix(units: &I, hexadecimal: bool) -> Option<(I, char, usize)> +where + I: Clone + Iterator, +{ + let mut trial = units.clone(); + consume_expected(&mut trial, '#')?; + if hexadecimal { + consume_expected(&mut trial, 'x')?; + } + + let radix = if hexadecimal { 16 } else { 10 }; + let mut value = 0_u32; + let mut has_digits = false; + loop { + let item = trial.next()?; + let digit = if (hexadecimal && item.ch.is_ascii_hexdigit()) + || (!hexadecimal && item.ch.is_ascii_digit()) + { + item.ch.to_digit(radix) + } else { + None + }; + if let Some(digit) = digit { + has_digits = true; + value = value.checked_mul(radix)?.checked_add(digit)?; + } else if has_digits && item.ch == ';' { + return Some((trial, char::from_u32(value)?, item.end)); + } else { + return None; + } + } +} + +fn consume_expected(units: &mut impl Iterator, expected: char) -> Option<()> { + (units.next()?.ch == expected).then_some(()) +} + +pub(crate) fn is_js_whitespace(ch: char) -> bool { + matches!( + ch, + '\u{0009}'..='\u{000d}' + | '\u{0020}' + | '\u{00a0}' + | '\u{1680}' + | '\u{2000}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + | '\u{feff}' + ) +} + +pub(crate) fn find_byte(bytes: &[u8], needle: u8, start: usize) -> Option { + let relative = bytes + .get(start..)? + .iter() + .position(|byte| *byte == needle)?; + Some(start.saturating_add(relative)) +} + +pub(crate) fn find_bytes(bytes: &[u8], needle: &[u8], start: usize) -> Option { + if needle.is_empty() { + return Some(start.min(bytes.len())); + } + let haystack = bytes.get(start..)?; + let relative = haystack + .windows(needle.len()) + .position(|window| window == needle)?; + Some(start.saturating_add(relative)) +} + +pub(crate) fn starts_with_at(bytes: &[u8], start: usize, needle: &[u8]) -> bool { + bytes + .get(start..) + .is_some_and(|tail| tail.starts_with(needle)) +} + +fn structural_tag<'a>(start: usize, token: &'a str, mode: ScanMode) -> Option> { + let content = token.strip_prefix('<')?; + let (kind, content) = if let Some(content) = content.strip_prefix('/') { + (TagKind::Closing, content) + } else { + let content = match mode { + ScanMode::Canonical => content.trim_start_matches(is_js_whitespace), + ScanMode::Structural => content, + }; + (TagKind::Opening, content) + }; + let name = take_tag_name(content)?; + Some(StructuralTag { + start, + end: start.checked_add(token.len())?, + name, + kind, + token, + }) +} + +pub(crate) fn is_structural_element_start(bytes: &[u8], start: usize) -> bool { + match bytes.get(start + 1).copied() { + Some(byte) if byte.is_ascii_alphabetic() => true, + Some(b'/') => bytes.get(start + 2).is_some_and(u8::is_ascii_alphabetic), + _ => false, + } +} + +pub(crate) fn raw_close_name_end(bytes: &[u8], start: usize, expected: &str) -> Option { + if !starts_with_at(bytes, start, b"')) + .then_some(name_end) +} + +pub(crate) fn take_tag_name(content: &str) -> Option<&str> { + let end = content + .char_indices() + .find(|(_, ch)| ch.is_ascii_whitespace() || matches!(*ch, '/' | '>')) + .map_or(content.len(), |(index, _)| index); + (end > 0).then(|| content.get(..end)).flatten() +} + +fn is_canonical_ignored(name: &str) -> bool { + TEXT_IGNORED.iter().any(|expected| { + let Some(prefix) = name.get(..expected.len()) else { + return false; + }; + prefix.eq_ignore_ascii_case(expected) + && name + .as_bytes() + .get(expected.len()) + .is_none_or(|byte| !byte.is_ascii_alphanumeric() && *byte != b'_') + }) +} + +fn contains_tag(tags: &[&str], name: &str) -> bool { + tags.iter() + .any(|candidate| name.eq_ignore_ascii_case(candidate)) +} + +/// Matches edge-core's `/\/\s*>$/` test on a complete tag token. +fn is_self_closing(token: &str) -> bool { + token + .strip_suffix('>') + .is_some_and(|body| body.trim_end_matches(is_js_whitespace).ends_with('/')) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn canonical_text(html: &str) -> String { + visible_text(html, usize::MAX).0 + } + + fn title(html: &str) -> Option { + let metadata = document_metadata(html); + title_from_range(html, metadata.title_range) + } + + #[test] + fn quoted_markup_and_unmatched_script_tags_stay_inert() { + for html in [ + r#"
Visible
"#, + r#"
Visible
"#, + ] { + assert_eq!(document_metadata(html).head_close, html.rfind("")); + assert_eq!(canonical_text(html), "Visible"); + } + } + + #[test] + fn browser_end_tag_and_comment_forms_close_where_browsers_do() { + // `` ends the script and `` is a whole comment, so each + // later `` sits inside an attribute value. + for prefix in [ + r#""#, + r#""#, + ] { + let html = format!("{prefix}
Visible
"); + assert_eq!(document_metadata(&html).head_close, Some(prefix.len())); + } + assert_eq!( + canonical_text("abce"#, + r#""#, + r#""#, + r#""#, + r#""#, + ); + let base = super::effective_document_base(source.as_bytes(), &document).expect("base"); + assert_eq!( + base.as_str(), + "https://cdn.example.net/npm/lib/?x=1&y=%2f#f" + ); + let invalid = super::effective_document_base( + br#""#, + &document, + ) + .expect("invalid first base falls back"); + assert_eq!(invalid, document); + } + + #[test] + fn decoded_query_bytes_are_serialized_once_in_each_rehosted_attribute() { + let value = "https://cdn.example.net/npm/a.png?q=%2f&literal=&amp;#f%2F"; + let page = rendered( + &rehosting_settings(), + &format!( + r#""# + ), + ); + let emitted = format!( + "https://{REQUEST_HOST}{PAGE_PREFIX}{ROUTE}a.png?q=%2f&literal=&amp;#f%2F" + ); + assert!(page.contains(&format!(r#"src="{emitted}""#)), "{page}"); + assert!( + page.contains(&format!(r#"srcset="{emitted} 1x""#)), + "{page}" + ); + assert!( + page.contains(&format!(r#"imagesrcset="{emitted} 2x""#)), + "{page}" + ); + } +} diff --git a/crates/trusted-server-core/src/html_processor/source_protection.rs b/crates/trusted-server-core/src/html_processor/source_protection.rs new file mode 100644 index 0000000000..7e926e0ad7 --- /dev/null +++ b/crates/trusted-server-core/src/html_processor/source_protection.rs @@ -0,0 +1,396 @@ +//! Bounded original-source hold around the entire HTML transformation pass. +//! Classification belongs to the selected middleware, not to core. + +use std::io; +use std::ops::Range; + +use futures::future::LocalBoxFuture; + +use crate::middleware::{MiddlewareChain, SourceProtection}; +use crate::platform::RuntimeServices; +use crate::streaming_processor::StreamProcessor; + +pub(super) struct SourceProtected

{ + inner: P, + classifiers: Vec, + original: Vec, + limit: usize, + ended: bool, +} + +impl

SourceProtected

{ + pub(super) fn new(inner: P, chains: &[&MiddlewareChain], limit: usize) -> Self { + Self { + inner, + classifiers: chains + .iter() + .flat_map(|chain| chain.source_protections()) + .collect(), + original: Vec::new(), + limit, + ended: false, + } + } + + fn hold(&mut self, chunk: &[u8]) -> io::Result<()> { + if self.ended { + return Err(refuse("source protection processor already ended")); + } + append(&mut self.original, chunk, self.limit) + } + + fn mask(&mut self) -> io::Result { + self.ended = true; + let mut ranges: Vec> = Vec::new(); + for classify in &self.classifiers { + let classified = classify(&self.original)?; + let mut previous_end = 0; + for range in &classified { + if range.start < previous_end + || range.start >= range.end + || range.end > self.original.len() + { + return Err(refuse("source protection returned invalid ranges")); + } + previous_end = range.end; + } + ranges.extend(classified); + } + ranges.sort_unstable_by_key(|range| (range.start, range.end)); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + if let Some(last) = merged.last_mut().filter(|last| range.start <= last.end) { + last.end = last.end.max(range.end); + } else { + merged.push(range); + } + } + MaskedDocument::new(std::mem::take(&mut self.original), merged, self.limit) + } +} + +impl StreamProcessor for SourceProtected

{ + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> io::Result> { + if self.classifiers.is_empty() { + return self.inner.process_chunk(chunk, is_last); + } + self.hold(chunk)?; + if !is_last { + return Ok(Vec::new()); + } + let masked = self.mask()?; + let input = masked.input(self.limit)?; + let output = self.inner.process_chunk(&input, true)?; + masked.restore(output, self.limit) + } + + fn finish<'a>( + &'a mut self, + chunk: &'a [u8], + services: &'a RuntimeServices, + ) -> LocalBoxFuture<'a, io::Result>> { + Box::pin(async move { + if self.classifiers.is_empty() { + return self.inner.finish(chunk, services).await; + } + self.hold(chunk)?; + let masked = self.mask()?; + let input = masked.input(self.limit)?; + let output = self.inner.finish(&input, services).await?; + masked.restore(output, self.limit) + }) + } + + fn reset(&mut self) { + self.inner.reset(); + self.original.clear(); + self.ended = false; + } +} + +struct MaskedDocument { + // Retain one original allocation. Protected SVGs remain slices, never copies. + original: Vec, + ranges: Vec>, + prefix: String, +} + +impl MaskedDocument { + fn new(original: Vec, ranges: Vec>, limit: usize) -> io::Result { + if original.len() > limit { + return Err(refuse("source protection original exceeds body limit")); + } + let prefix = if ranges.is_empty() { + String::new() + } else { + format!("", self.prefix) + } + + fn input(&self, limit: usize) -> io::Result> { + if self.ranges.is_empty() { + return Ok(std::borrow::Cow::Borrowed(&self.original)); + } + let mut input = Vec::new(); + let mut cursor = 0; + for (index, range) in self.ranges.iter().enumerate() { + append(&mut input, &self.original[cursor..range.start], limit)?; + append(&mut input, self.marker(index).as_bytes(), limit)?; + cursor = range.end; + } + append(&mut input, &self.original[cursor..], limit)?; + Ok(std::borrow::Cow::Owned(input)) + } + + fn restore(&self, output: Vec, limit: usize) -> io::Result> { + if output.len() > limit { + return Err(refuse("source protection output exceeds body limit")); + } + if self.ranges.is_empty() { + return Ok(output); + } + let mut restored = Vec::new(); + let mut cursor = 0; + for (index, range) in self.ranges.iter().enumerate() { + let at = find(&output[cursor..], self.prefix.as_bytes()) + .map(|at| cursor + at) + .ok_or_else(|| refuse("source protection marker lost"))?; + let marker = self.marker(index); + if !output[at..].starts_with(marker.as_bytes()) { + return Err(refuse( + "source protection marker duplicated, reordered or altered", + )); + } + append(&mut restored, &output[cursor..at], limit)?; + append(&mut restored, &self.original[range.clone()], limit)?; + cursor = at + marker.len(); + } + if find(&output[cursor..], self.prefix.as_bytes()).is_some() { + return Err(refuse("source protection marker duplicated")); + } + append(&mut restored, &output[cursor..], limit)?; + Ok(restored) + } +} + +fn append(output: &mut Vec, bytes: &[u8], limit: usize) -> io::Result<()> { + if bytes.len() > limit.saturating_sub(output.len()) { + return Err(refuse("source protection exceeds body limit")); + } + // Reserve explicitly so Vec's implicit growth cannot overshoot the bound. + let needed = output.len() + bytes.len(); + if needed > output.capacity() { + let capacity = output.capacity().saturating_mul(2).max(needed).min(limit); + output.reserve_exact(capacity - output.len()); + } + output.extend_from_slice(bytes); + Ok(()) +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn refuse(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::middleware::{ + HTML_MEDIA_TYPE, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, + }; + use crate::platform::test_support::noop_services; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct Policy(SourceProtection); + + impl Middleware for Policy { + fn middleware_id(&self) -> &'static str { + "test.protection" + } + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + fn source_protection(&self) -> Option { + Some(self.0) + } + fn create(&self, _: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } + } + + struct Replace { + calls: Arc, + transform: fn(&[u8]) -> Vec, + } + + impl StreamProcessor for Replace { + fn process_chunk(&mut self, chunk: &[u8], _: bool) -> io::Result> { + self.calls.fetch_add(1, Ordering::Relaxed); + Ok((self.transform)(chunk)) + } + } + + fn processor( + classify: SourceProtection, + transform: fn(&[u8]) -> Vec, + limit: usize, + ) -> (SourceProtected, Arc) { + let chain = MiddlewareChain::new( + MiddlewarePhase::Serve, + HTML_MEDIA_TYPE, + vec![Arc::new(Policy(classify))], + ); + let calls = Arc::new(AtomicUsize::new(0)); + ( + SourceProtected::new( + Replace { + calls: Arc::clone(&calls), + transform, + }, + &[&chain], + limit, + ), + calls, + ) + } + + fn middle(_: &[u8]) -> io::Result>> { + Ok(std::iter::once(1..5).collect()) + } + fn unchanged(bytes: &[u8]) -> Vec { + bytes.to_vec() + } + + #[test] + fn holds_original_until_complete_and_restores_exact_bytes_after_async_finish() { + let (mut processor, calls) = processor( + middle, + |bytes| { + let mut output = b"outside".to_vec(); + output.extend_from_slice(&bytes[1..]); + output + }, + 1024, + ); + assert!( + processor + .process_chunk(b"aKEEP", false) + .expect("hold") + .is_empty() + ); + assert_eq!(calls.load(Ordering::Relaxed), 0, "no early transform"); + let result = + futures::executor::block_on(processor.finish(b"z", &noop_services())).expect("restore"); + assert_eq!(result, b"outsideKEEPz"); + assert_eq!(calls.load(Ordering::Relaxed), 1); + } + + #[test] + fn source_limit_is_checked_before_copying_or_running_transforms() { + let (mut processor, calls) = processor(middle, unchanged, 5); + assert!( + processor + .process_chunk(b"a", false) + .expect("hold") + .is_empty() + ); + let before = processor.original.capacity(); + assert!(processor.process_chunk(&[b'x'; 128], false).is_err()); + assert_eq!(processor.original, b"a"); + assert_eq!(processor.original.capacity(), before); + assert_eq!(calls.load(Ordering::Relaxed), 0); + } + + #[test] + fn refuses_invalid_ranges_without_exposing_output() { + for classify in [ + (|_: &[u8]| Ok(std::iter::once(4..7).collect())) as SourceProtection, + |_: &[u8]| Ok(std::iter::once(2..2).collect()), + |_: &[u8]| Ok(vec![1..4, 3..5]), + |_: &[u8]| Ok(vec![3..5, 0..1]), + |_: &[u8]| Err(refuse("classification failed")), + ] { + let (mut processor, calls) = processor(classify, unchanged, 1024); + assert!(processor.process_chunk(b"aKEEPz", true).is_err()); + assert_eq!(calls.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn refuses_lost_duplicated_reordered_and_altered_markers() { + let masked = + MaskedDocument::new(b"aKEEPzOTHERq".to_vec(), vec![1..5, 6..11], 1024).expect("mask"); + let first = masked.marker(0); + let second = masked.marker(1); + for invalid in [ + format!("a{second}q"), + format!("a{first}{first}{second}q"), + format!("a{second}{first}q"), + format!("a{}x-->{second}q", masked.prefix), + format!("a{first}{second}{second}q"), + ] { + assert!(masked.restore(invalid.into_bytes(), 1024).is_err()); + } + assert_eq!( + masked + .restore(format!("edited{first}z{second}q").into_bytes(), 1024) + .expect("restore"), + b"editedKEEPzOTHERq" + ); + } + + #[test] + fn refuses_mask_and_restoration_expansion_past_limit() { + let short = MaskedDocument::new(b"aKEEPz".to_vec(), std::iter::once(1..5).collect(), 6) + .expect("original fits"); + assert!(short.input(6).is_err(), "opaque marker must fit too"); + let original = format!("a{}z", "K".repeat(128)).into_bytes(); + let limit = original.len(); + let masked = MaskedDocument::new( + original.clone(), + std::iter::once(1..limit - 1).collect(), + limit, + ) + .expect("fits"); + let input = masked.input(limit).expect("mask fits").into_owned(); + assert!(masked.restore(input.clone(), limit - 1).is_err()); + assert_eq!(masked.restore(input, limit).expect("exact limit"), original); + } + + #[test] + fn policy_is_default_off_and_selected_only_from_supplied_chains() { + let chain = MiddlewareChain::new(MiddlewarePhase::Serve, HTML_MEDIA_TYPE, Vec::new()); + let calls = Arc::new(AtomicUsize::new(0)); + let mut processor = SourceProtected::new( + Replace { + calls: Arc::clone(&calls), + transform: unchanged, + }, + &[&chain], + 1, + ); + assert_eq!( + processor.process_chunk(b"unheld", false).expect("stream"), + b"unheld" + ); + assert_eq!(calls.load(Ordering::Relaxed), 1); + } +} diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs index 69d9fca9ab..2e9b90a296 100644 --- a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -684,9 +684,13 @@ mod tests { vec![Arc::new(ScriptAddress(integration))], ); let config = HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: None, + response_headers: None, public_prefix: None, prepared: std::sync::Arc::new(crate::middleware::Prepared::default()), + status: http::StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "publisher.example.com".to_string(), diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index ea672660ff..01b0220613 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -22,6 +22,7 @@ use crate::error::TrustedServerError; use crate::evidence::OwnedRequestInfo; use crate::geo::GeoInfo; use crate::http_util::is_navigation_request; +use crate::media_signing::MediaSigner; use crate::middleware::{Middleware, MiddlewareChain, MiddlewarePhase, PhaseEntries}; use crate::module_context::{ModuleCall, ModuleContext, ModuleRequest, ResolvedRequest}; use crate::origin_auth::OriginSigners; @@ -707,6 +708,11 @@ pub struct IntegrationRegistration { /// on every request, so declaring one runs it wherever the registration /// itself runs. pub identifier_module: Option>, + /// Media signer this registration supplies, see [`crate::media_signing`]. + /// + /// No section selects one. It runs wherever the registration itself + /// runs, and at most one running registration may supply one. + pub media_signer: Option>, } impl IntegrationRegistration { @@ -741,6 +747,7 @@ impl IntegrationRegistrationBuilder { source_module: None, analytics_module: None, identifier_module: None, + media_signer: None, }, } } @@ -872,6 +879,18 @@ impl IntegrationRegistrationBuilder { self } + /// Declare the media signer this registration supplies, which core asks + /// to sign the binary responses it claims, see [`crate::media_signing`]. + /// + /// It runs wherever the registration runs. The registry refuses to build + /// when two running registrations supply one, because signing a body + /// twice would sign the first signer's output as a source. + #[must_use] + pub fn with_media_signer(mut self, signer: Arc) -> Self { + self.registration.media_signer = Some(signer); + self + } + /// Declare the rules this registration adds to `/robots.txt`, under their /// `name`. /// @@ -1025,6 +1044,9 @@ struct IntegrationRegistryInner { // registration order, which core asks on every request. Shared, so a // request's services take a reference count rather than a copy. identifier_modules: Arc<[Arc]>, + // The media signer a running registration supplies, with the + // integration that supplied it. At most one. + media_signer: Option<(&'static str, Arc)>, } /// The sources a deployment's settings name, built once. @@ -1074,6 +1096,7 @@ impl Default for IntegrationRegistryInner { analytics_module: None, sources: None, identifier_modules: Arc::default(), + media_signer: None, } } } @@ -2304,6 +2327,19 @@ impl IntegrationRegistry { )?; identifier_modules.push(module); } + if let Some(signer) = registration.media_signer { + if let Some((supplier, _)) = &inner.media_signer { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integrations `{supplier}` and `{}` both supply a media signer, and \ + at most one running integration may, because a body is signed \ + once. Select only one of their modules", + registration.integration_id + ), + })); + } + inner.media_signer = Some((registration.integration_id, signer)); + } if let Some((name, contributor)) = registration.robots_txt_contributor { claim_module_name( &mut claimed, @@ -2451,6 +2487,17 @@ impl IntegrationRegistry { Arc::clone(&self.inner.identifier_modules) } + /// The media signer the running registrations supply, if one does. + /// Adapters hand it to each request's services with + /// [`RuntimeServices::with_media_signer`]. + #[must_use] + pub fn media_signer(&self) -> Option> { + self.inner + .media_signer + .as_ref() + .map(|(_, signer)| Arc::clone(signer)) + } + /// The prefixes of the publisher `request` was resolved to under /// `settings`, from the selected prefix module. /// @@ -3059,6 +3106,7 @@ impl IntegrationRegistry { analytics_module: None, sources: None, identifier_modules: Arc::default(), + media_signer: None, }), plan: None, platform: Arc::default(), @@ -3147,6 +3195,7 @@ impl IntegrationRegistry { analytics_module: None, sources: None, identifier_modules: Arc::default(), + media_signer: None, }), plan: None, platform: Arc::default(), @@ -4070,6 +4119,35 @@ pub(crate) mod test_support { } } + /// Ends each stylesheet a `[[serve]]` entry names it for with a line, + /// standing in for a stylesheet signer. + pub(crate) const STYLESHEET: &str = "testing.middleware-fixture.stylesheet"; + /// The line [`STYLESHEET`] adds. + pub(crate) const STYLESHEET_LINE: &str = "/* served by the middleware fixture */\n"; + + struct Stylesheet; + + impl Middleware for Stylesheet { + fn middleware_id(&self) -> &'static str { + STYLESHEET + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == "text/css" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + stream: Some(Box::new(AppendLine(STYLESHEET_LINE))), + ..MiddlewareAction::pass() + } + } + } + /// Says it moves the origin's address itself, on its own media type, /// and then leaves the body as it came with a line added, so a test /// sees what core left to it. @@ -4149,6 +4227,7 @@ pub(crate) mod test_support { .with_middleware(Arc::new(Locating)) .with_middleware(Arc::new(Labeled { text: labeled })) .with_middleware(Arc::new(Framed)) + .with_middleware(Arc::new(Stylesheet)) .build(), )) } @@ -4552,6 +4631,7 @@ mod tests { }; use super::*; use crate::constants::COOKIE_EC; + use crate::media_signing::test_support::{Behavior, StubSigner}; use crate::middleware::{HTML_MEDIA_TYPE, MiddlewareAction, MiddlewareContext, PhaseEntry}; use crate::permissions::{Permission, PermissionSet, PermissionState}; use crate::platform::test_support::noop_services; @@ -7687,6 +7767,101 @@ mod tests { ); } + /// The names the two media signer test modules run under. + const MEDIA_PROBE_MODULE: &str = "testing.media-probe"; + const MEDIA_SECOND_MODULE: &str = "testing.media-second"; + + fn media_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("media-probe") + .with_media_signer(StubSigner::new(Behavior::Sign, 1024)) + .build(), + )) + } + + fn media_second_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("media-second") + .with_media_signer(StubSigner::new(Behavior::Sign, 1024)) + .build(), + )) + } + + fn media_probe_builder() -> crate::integrations::IntegrationBuilder { + crate::integrations::IntegrationBuilder::new( + "media-probe", + "seam-probe", + media_probe_registration, + validate_nothing, + ) + .with_module_name(MEDIA_PROBE_MODULE) + } + + /// No section selects a media signer. A running registration's is handed + /// out for each request's services. + #[test] + fn a_running_registration_hands_out_its_media_signer() { + let settings = settings_naming(MEDIA_PROBE_MODULE); + + let registry = IntegrationRegistry::with_registrations(&settings, &[media_probe_builder()]) + .expect("should build a registry with a media signer"); + + assert_eq!( + registry.media_signer().map(|signer| signer.signer_id()), + Some("testing.media"), + "should hand out the running registration's signer" + ); + } + + /// A registration a section does not select runs nothing, its media + /// signer included, so no response is claimed. + #[test] + fn an_unselected_registration_offers_no_media_signer() { + let settings = crate::test_support::tests::create_test_settings(); + + let registry = IntegrationRegistry::with_registrations(&settings, &[media_probe_builder()]) + .expect("should build a registry without the probe selected"); + + assert!( + registry.media_signer().is_none(), + "an unselected registration's signer should not be handed out" + ); + } + + /// Two running signers would sign one signer's output as the other's + /// source, so the pair is refused. + #[test] + fn two_running_registrations_supplying_a_media_signer_are_refused() { + let mut settings = settings_naming(MEDIA_PROBE_MODULE); + settings.select_module("testing", MEDIA_SECOND_MODULE); + let extra = [ + media_probe_builder(), + crate::integrations::IntegrationBuilder::new( + "media-second", + "seam-probe", + media_second_registration, + validate_nothing, + ) + .with_module_name(MEDIA_SECOND_MODULE), + ]; + + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should refuse two media signers"); + + let message = error.to_string(); + assert!( + message.contains("both supply a media signer") + && message.contains("`media-probe`") + && message.contains("`media-second`"), + "error should name both integrations: {message}" + ); + } + fn html_entry(path: Option<&str>, names: &[&str]) -> PhaseEntry { PhaseEntry { media_type: HTML_MEDIA_TYPE.to_owned(), @@ -7712,51 +7887,6 @@ mod tests { let registry = IntegrationRegistry::new(&settings) .expect("should build a registry whose entries name registered middleware"); - assert_eq!( - registry.middleware_ids(), - [ - fixture::HEAD, - fixture::LINKS, - fixture::BROKEN, - fixture::READER, - fixture::BROKEN_READER, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::TEXT_HEADER, - fixture::LOCATING, - fixture::LABELED, - fixture::FRAMED, - crate::service_version::MODULE, - ], - "should list the middleware the selected module supplies, and core's own after them" - ); - assert_eq!( - registry.middleware_in(MiddlewarePhase::Fetch), - [ - fixture::HEAD, - fixture::LINKS, - fixture::BROKEN, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::LABELED, - fixture::FRAMED - ], - "should list the middleware that run in the fetch phase" - ); - assert_eq!( - registry.middleware_in(MiddlewarePhase::Serve), - [ - fixture::READER, - fixture::BROKEN_READER, - fixture::PREPARING, - fixture::TEXT_HEADER, - fixture::LOCATING, - crate::service_version::MODULE - ], - "should list the middleware that run in the serve phase, core's own last" - ); let chain_for = |media_type: &str, path: &str| { registry .middleware_chain(&settings.fetch, MiddlewarePhase::Fetch, media_type, path) @@ -7979,45 +8109,6 @@ mod tests { ); } - #[test] - fn an_entry_naming_a_middleware_nothing_supplies_is_refused_with_what_is_supplied() { - let settings = fixture_settings_with_entries(vec![ - html_entry(Some("/news/"), &[fixture::HEAD]), - html_entry(None, &[fixture::HEAD, "testing.nothing"]), - ]); - - let error = IntegrationRegistry::new(&settings) - .err() - .expect("should refuse an entry naming a middleware nothing supplies"); - - let message = error.to_string(); - assert!( - message.contains( - "[[fetch]] entry 2 names `testing.nothing`, which no module that runs supplies" - ) && message.contains(&format!( - "[{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}]", - fixture::HEAD, - fixture::LINKS, - fixture::BROKEN, - fixture::READER, - fixture::BROKEN_READER, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::TEXT_HEADER, - fixture::LOCATING, - fixture::LABELED, - fixture::FRAMED, - crate::service_version::MODULE - )), - "should name the entry and list what could be named: {message}" - ); - assert!( - !message.contains("no section selects"), - "should not blame a selection when the name is no module's: {message}" - ); - } - #[test] fn an_entry_naming_the_middleware_of_a_module_no_section_selects_says_so() { let mut settings = crate::test_support::tests::create_test_settings(); @@ -8159,59 +8250,6 @@ mod tests { ); } - #[test] - fn a_middleware_no_entry_names_is_listed_for_the_startup_warning() { - let unnamed = |entries: Vec| -> Vec<&'static str> { - let settings = fixture_settings_with_entries(entries); - let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); - unnamed_middleware(&settings, ®istry.inner) - .into_iter() - .map(|(integration, middleware)| { - assert_eq!(integration, fixture::ID, "should name the supplier"); - middleware.middleware_id() - }) - .collect() - }; - - assert_eq!( - unnamed(Vec::new()), - [ - fixture::HEAD, - fixture::LINKS, - fixture::BROKEN, - fixture::READER, - fixture::BROKEN_READER, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::TEXT_HEADER, - fixture::LOCATING, - fixture::LABELED, - fixture::FRAMED, - ], - "should list every middleware when there are no entries" - ); - assert_eq!( - unnamed(vec![ - html_entry(Some("/news/"), &[fixture::LINKS]), - html_entry(None, &[fixture::HEAD]), - ]), - [ - fixture::BROKEN, - fixture::READER, - fixture::BROKEN_READER, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::TEXT_HEADER, - fixture::LOCATING, - fixture::LABELED, - fixture::FRAMED - ], - "should leave out a middleware any entry names" - ); - } - #[test] fn an_entry_naming_a_middleware_in_a_phase_it_does_not_run_in_is_refused() { let mut settings = settings_naming(fixture::MODULE); @@ -8264,26 +8302,6 @@ mod tests { chain_for(MiddlewarePhase::Serve, "/sport/today").is_empty(), "should run no serve middleware on a path only the fetch entries cover" ); - let unnamed: Vec<&str> = unnamed_middleware(&settings, ®istry.inner) - .into_iter() - .map(|(_, middleware)| middleware.middleware_id()) - .collect(); - assert_eq!( - unnamed, - [ - fixture::LINKS, - fixture::BROKEN, - fixture::BROKEN_READER, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::TEXT_HEADER, - fixture::LOCATING, - fixture::LABELED, - fixture::FRAMED - ], - "should count a middleware as named only by an entry of a phase it runs in" - ); } /// A middleware under a name of the test's choosing, running in the diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 0422b2882c..f33e9d7361 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -45,6 +45,7 @@ pub mod constants; pub mod cookies; pub mod creative; pub mod creative_opportunities; +pub(crate) mod css_relocation; pub mod ec; pub mod edge_cookie; pub mod error; @@ -56,6 +57,7 @@ pub mod html_processor; pub mod http_util; pub mod inspect; pub mod integrations; +pub mod media_signing; pub mod middleware; pub mod module_context; pub mod module_name; @@ -70,6 +72,7 @@ pub mod provider_table; pub mod proxy; pub mod publisher; pub mod redacted; +pub mod rehost; pub mod request_signing; pub(crate) mod response_header_rewrite; pub mod response_privacy; diff --git a/crates/trusted-server-core/src/media_signing.rs b/crates/trusted-server-core/src/media_signing.rs new file mode 100644 index 0000000000..82c7c7ff4f --- /dev/null +++ b/crates/trusted-server-core/src/media_signing.rs @@ -0,0 +1,2094 @@ +//! Signed media, which a module makes from the exact bytes of a binary +//! response on its way to a reader. +//! +//! A module supplies a [`MediaSigner`] through +//! [`IntegrationRegistrationBuilder::with_media_signer`](crate::integrations::IntegrationRegistrationBuilder::with_media_signer), +//! and it runs wherever the registration runs. Core asks it about two +//! paths: a binary response the publisher path passes through, and the +//! answer of an asset route. Which responses are signed is decided by the +//! origin's media type, so a path with no file extension is covered as an +//! image, audio or video file is. +//! +//! The origin is asked as the reader asked, so a response that is not +//! signed keeps the origin's own range and conditional handling. Where the +//! origin answers `200` with a type the signer handles, within its limit, +//! core captures the body before anything else reads it and hands those +//! exact bytes to the signer. A request that plausibly asks for media asks +//! the origin for an identity-coded body, the only coding signed. Where the +//! origin answers a `206` or a `HEAD` for such a type, core asks it once +//! more for the whole representation, unconditionally, and signs what that +//! answer carries, but only where the signed whole can be kept, so the +//! origin is not read whole for every range. Where the reader's range or +//! `If-Match` brings a `416` or `412` the longer signed representation may +//! answer, and where the reader's `If-Range` or `If-Match` names a signed +//! representation, the origin is asked for the whole representation too. +//! Core applies no transformation of its own, so the captured source is +//! also the final body. +//! +//! An asset route that uses the platform's image optimizer is the one +//! place a transformation happens, and the optimizer reads the origin +//! itself, so core never holds its input. Core asks the optimizer to keep +//! its input's C2PA manifest and add a manifest of its own recording what +//! it did, which binds the input it read by hash. Only a whole `200` to a +//! `GET` whose body carries a C2PA manifest store is handed to the signer, +//! marked as [`MediaJob::optimizer_output`], so the optimizer's own record +//! is the parent of what is signed and core adds nothing to the history. +//! An optimizer writes such a record only where its input carried a +//! manifest, so an unsigned original's optimized answer is never signed. +//! A route may instead send the optimizer to its own signed original on +//! the serving host, see `AssetImageOptimizerConfig::signed_source`, so +//! the optimizer's input is the exact origin bytes core captured and +//! signed. +//! Nothing of it is kept, since its bytes depend on the transformation and +//! the reader's `Accept`, and its signed answer says it varies by `Accept`. +//! +//! The signed bytes are a representation of their own. Core gives them a +//! strong `ETag` of their own digest, a `Repr-Digest` and a +//! `Content-Length`, and answers the reader's `If-Match`, `If-None-Match`, +//! `If-Range` and `Range` against them, with `304`, `412`, `206` and `416` +//! as each requires. The origin's validators never describe signed bytes, +//! so they are removed. Ranges are offered only on a signed representation +//! that was kept. +//! +//! A signed representation is kept in the `[cache.assets]` loading cache +//! for as long as the origin's own headers made it shareably fresh, under +//! the namespace the signer signed it under, and only for a request whose +//! answer the origin could not have made one reader's own. Before the +//! origin is asked, a request of that kind which plausibly asks for media +//! and asks for no revalidation is looked up under the signer's current +//! [`namespace`](MediaSigner::namespace), so a changed or revoked policy +//! never serves a kept copy. A kept copy carries its `Age`, and one whose +//! body no longer hashes to its tag is dropped. +//! +//! What is never signed: a response the origin marked as one reader's, a +//! request carrying `Authorization`, a body larger than the signer's limit, +//! a content-coded body, a body the origin forbade transforming, a +//! response whose own `Link` already names a C2PA manifest, read from the +//! origin's own headers before core writes a cache policy, and a captured +//! body that is an init or media segment of a fragmented ISO BMFF stream, +//! whose provenance binds the stream's parts together. Such a response +//! goes to the reader as the origin answered it, its `Link` untouched. +//! One core meant to +//! sign and could not goes out without the origin's validators and held +//! only briefly and privately, so the next request can be signed. +//! +//! A signer may also sign a source larger than its limit, unchanged, in +//! the large lane, see [`MediaSigner::sign_large`]. Core never holds such a +//! source whole: it pins one version by the origin's strong `ETag` and +//! length, hashes it in bounded chunks, hands the signer a +//! [`VersionedMediaSource`] whose every read is pinned with `If-Match`, and +//! answers the reader with the origin's own bytes and validators plus the +//! signer's headers and a `Repr-Digest`. Only metadata is kept, under the +//! signer's current namespace and the pinned tag. A source without a strong +//! `ETag` goes to the reader as the origin answered it. + +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use base64::Engine as _; +use bytes::{Bytes, BytesMut}; +use edgezero_core::body::Body as EdgeBody; +use futures::StreamExt as _; +use http::{HeaderMap, HeaderName, HeaderValue, Method, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; +use web_time::{SystemTime, UNIX_EPOCH}; + +use crate::middleware::{ResponseHeader, media_type_of}; +use crate::platform::{CacheSpec, LoadError, LoadedValue, RuntimeServices, StoreKey}; +use crate::response_privacy::cache_control_forbids_shared_storage; +use crate::settings::{ASSETS_CACHE, Settings}; + +mod large; + +pub(crate) use large::{LargeAnswer, PinnedOrigin}; +pub use large::{ + LargeMediaJob, LargeMediaSigning, LargeSignedMedia, MediaSourceError, SourceVersion, + VersionedMediaSource, +}; + +/// A module that signs the exact bytes of a binary representation. +/// +/// Registered with +/// [`IntegrationRegistrationBuilder::with_media_signer`](crate::integrations::IntegrationRegistrationBuilder::with_media_signer). +/// At most one running registration may supply one. +pub trait MediaSigner: Send + Sync { + /// The module's name, for logs. + fn signer_id(&self) -> &'static str; + + /// Whether a `GET` or `HEAD` request for `path` may be signed, asked + /// before the origin is. Core changes nothing about the origin request, + /// so this only leaves out paths that never carry media, such as an API. + /// The media type of the origin's answer decides the rest, so a signer + /// answers `true` for a path with no file extension. + fn handles_path(&self, path: &str) -> bool; + + /// Whether this signer signs a body of `media_type`, a bare lower case + /// `type/subtype` read from the origin's `Content-Type`. + fn handles_media_type(&self, media_type: &str) -> bool; + + /// The largest body core captures for this signer. A larger one goes to + /// the reader as the origin answered it, never held whole. + fn max_source_bytes(&self) -> usize; + + /// The policy and credential context a kept signed representation is + /// bound to, from a policy check no older than the signer's own policy + /// lifetime. Core keeps a signed representation under it and serves a + /// kept one only while this answers the same value. An error keeps + /// nothing and serves nothing kept for the request. Never a secret. + fn namespace<'a>(&'a self, services: &'a RuntimeServices) -> MediaNamespacing<'a>; + + /// Signs one captured representation. + fn sign<'a>(&'a self, services: &'a RuntimeServices, job: &'a MediaJob<'a>) + -> MediaSigning<'a>; + + /// The largest source this signer signs unchanged in the large lane, + /// see [`sign_large`](Self::sign_large), or `0`, the default, where it + /// signs none. Core uses the lane only for a source larger than + /// [`max_source_bytes`](Self::max_source_bytes) and no larger than this. + fn max_large_source_bytes(&self) -> u64 { + 0 + } + + /// The one wall-clock budget for a large source: core's hashing of it + /// and the whole of [`sign_large`](Self::sign_large). Core drops the + /// work, stalled reads included, when it is spent. `Duration::ZERO`, + /// the default, keeps the large lane closed, and the lane is also + /// closed on a platform without a timer. + fn max_large_media_duration(&self) -> Duration { + Duration::ZERO + } + + /// Signs one large source that reaches the reader unchanged, from its + /// whole-source digest and pinned reads, returning only the headers its + /// answers carry. + /// + /// Core asks only for a shared `GET` whose origin answer is a `200` or + /// `206` of a type the signer handles, identity coded, not marked as + /// one reader's, not forbidden from transformation, naming no manifest + /// of its own, saying `Accept-Ranges: bytes`, with one strong `ETag` + /// and a known total length above + /// [`max_source_bytes`](Self::max_source_bytes) and within + /// [`max_large_source_bytes`](Self::max_large_source_bytes), on a + /// platform that streams response bodies and offers a timer, and only + /// where no metadata is kept for that version under the current + /// [`namespace`](Self::namespace). The default signs nothing. + fn sign_large<'a>( + &'a self, + _services: &'a RuntimeServices, + _job: &'a LargeMediaJob<'a>, + ) -> LargeMediaSigning<'a> { + Box::pin(async { + Err(MediaSigningFailure { + reason: "large_media_unsupported", + }) + }) + } +} + +/// What [`MediaSigner::sign`] returns. +pub type MediaSigning<'a> = + Pin> + 'a>>; + +/// What [`MediaSigner::namespace`] returns. +pub type MediaNamespacing<'a> = + Pin> + 'a>>; + +/// One captured representation a signer is asked to sign. +#[derive(Debug, Clone, Copy)] +pub struct MediaJob<'a> { + /// The address the reader asked for, `scheme://host/path`, without its + /// query. + pub reader_url: &'a str, + /// The origin's `Content-Type`, as it sent it. + pub content_type: &'a str, + /// The exact body core received, captured before anything else read + /// it. Core transforms nothing, so it is also the final body: the + /// operations core performed are none and the record is complete. + pub source: &'a Bytes, + /// SHA-256 of [`source`](Self::source), lower case hex. + pub source_sha256: &'a str, + /// The address core fetched the source from, exactly as fetched, or + /// `None` when that address carries user information, a query or a + /// fragment, or when the source is + /// [`optimizer_output`](Self::optimizer_output). It names a mutable + /// resource, not an immutable version. + pub source_reference: Option<&'a str>, + /// The lower case host of the address core fetched the source from, + /// with no scheme, port, path, query or user information, present even + /// where [`source_reference`](Self::source_reference) is not. Core + /// always sets it for a body it captured over HTTP. For + /// [`optimizer_output`](Self::optimizer_output) it is the origin host + /// the optimizer was asked to read. + pub source_origin_host: Option<&'a str>, + /// The source is the platform image optimizer's answer, not the + /// origin's bytes. Core hands such a source over only where it carries + /// a C2PA manifest store, which the optimizer writes to record what it + /// did to its own input. A signer must have the signature refused + /// unless that embedded manifest is present and intact, so the + /// optimizer's own record of its input and transformations is the + /// parent of what is signed, never replaced by a claim of core's. + pub optimizer_output: bool, +} + +/// The signed representation a signer returns. +#[derive(Debug, Clone)] +pub struct SignedMedia { + /// The final signed bytes, which core sends as they are. + pub bytes: Bytes, + /// The `Content-Type` of the signed bytes. + pub content_type: HeaderValue, + /// Headers the signer writes on every response of the signed + /// representation: `200`, `206`, `304`, `412`, `416` and `HEAD`. A `Link` + /// is added to the origin's own links, and any other replaces the + /// origin's header of that name. A name core owns, see + /// [`is_reserved_header`], fails the signing. + pub headers: Vec, + /// The policy and credential context these bytes were actually signed + /// under, as [`MediaSigner::namespace`] gives it for that context. Core + /// keeps the signed representation under this value, never under a + /// namespace asked for again, so bytes signed under one policy are never + /// served while another is current. + pub namespace: String, +} + +/// Why a signer did not sign, which core logs before it sends the origin's +/// bytes unsigned. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +#[display("{reason}")] +pub struct MediaSigningFailure { + /// A short code naming the reason. + pub reason: &'static str, +} + +impl core::error::Error for MediaSigningFailure {} + +/// Headers core writes on a signed representation, which a signer may not +/// set. +const RESERVED_HEADERS: [&str; 17] = [ + "content-length", + "content-type", + "content-encoding", + "content-range", + "accept-ranges", + "etag", + "last-modified", + "cache-control", + "expires", + "vary", + "set-cookie", + "transfer-encoding", + "repr-digest", + "content-digest", + "digest", + "age", + "date", +]; + +/// Origin headers that describe the origin's bytes or one transfer of them, +/// never signed bytes. +const ORIGIN_REPRESENTATION_HEADERS: [&str; 11] = [ + "content-length", + "content-encoding", + "content-range", + "accept-ranges", + "etag", + "last-modified", + "repr-digest", + "content-digest", + "digest", + "content-md5", + "transfer-encoding", +]; + +/// Origin headers a kept signed representation carries besides the ones +/// core and the signer write. Core's per-request headers are written again +/// on each reader's copy, so none of them is kept. +const KEPT_ORIGIN_HEADERS: [&str; 6] = [ + "cache-control", + "expires", + "content-disposition", + "content-language", + "link", + "x-content-type-options", +]; + +/// Request headers that make an origin answer with less than the whole +/// representation. +const PRECONDITION_HEADERS: [HeaderName; 6] = [ + header::RANGE, + header::IF_RANGE, + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::IF_MATCH, + header::IF_UNMODIFIED_SINCE, +]; + +/// The first bytes of a kept signed representation, naming its layout. +const KEPT_MAGIC: &[u8; 4] = b"TSM2"; + +/// How many kept signed representations one process holds in memory. Each +/// may be as large as a signer's limit, so few are held. +const KEPT_MEMORY_ENTRIES: usize = 4; + +/// What starts the opaque part of a signed representation's entity tag, +/// so a reader's precondition naming signed bytes is told from one naming +/// the origin's. +const SIGNED_TAG_PREFIX: &str = "sig-"; + +/// The `Cache-Control` an answer core meant to sign and could not goes out +/// with, so no cache holds unsigned bytes in place of signed ones for long. +const UNSIGNED_FALLBACK_CACHE_CONTROL: &str = "private, max-age=60"; + +/// `Sec-Fetch-Dest` values a browser sends for media. +const MEDIA_DESTINATIONS: [&str; 7] = [ + "image", "audio", "video", "track", "font", "object", "embed", +]; + +/// File extensions of media, for a request that names neither a media +/// destination nor a media type it accepts. +const MEDIA_EXTENSIONS: [&str; 33] = [ + "jpg", "jpeg", "png", "gif", "webp", "avif", "heic", "heif", "tif", "tiff", "svg", "bmp", + "ico", "jxl", "mp4", "m4v", "mov", "m4a", "mp3", "wav", "aac", "flac", "ogg", "oga", "ogv", + "opus", "webm", "avi", "mkv", "woff", "woff2", "ttf", "otf", +]; + +/// What a lookup that finds nothing kept answers, never stored. +const NOT_KEPT: &str = "no signed representation kept"; + +/// Whether `name` is a header core owns on a signed representation. +#[must_use] +pub fn is_reserved_header(name: &HeaderName) -> bool { + RESERVED_HEADERS.contains(&name.as_str()) +} + +/// Removes every header that would make the origin answer with less than +/// the whole representation, for the second, whole fetch of a claim. +pub(crate) fn strip_representation_preconditions(headers: &mut HeaderMap) { + for name in &PRECONDITION_HEADERS { + headers.remove(name); + } +} + +/// What the reader's request asked of the representation. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct ReaderPreconditions { + if_match: Option, + if_none_match: Option, + if_range: Option, + range: Option, + if_modified_since: Option, + if_unmodified_since: Option, +} + +impl ReaderPreconditions { + fn of(headers: &HeaderMap) -> Self { + Self { + if_match: joined(headers, &header::IF_MATCH), + if_none_match: joined(headers, &header::IF_NONE_MATCH), + if_range: joined(headers, &header::IF_RANGE), + range: joined(headers, &header::RANGE), + if_modified_since: joined(headers, &header::IF_MODIFIED_SINCE), + if_unmodified_since: joined(headers, &header::IF_UNMODIFIED_SINCE), + } + } + + /// Whether `If-Range` or `If-Match` names a signed representation's + /// entity tag, see [`SIGNED_TAG_PREFIX`]. + fn names_signed_bytes(&self) -> bool { + let if_range = self + .if_range + .as_deref() + .and_then(single_entity_tag) + .is_some_and(|tag| is_signed_tag(&tag)); + let if_match = self + .if_match + .as_deref() + .is_some_and(|value| match TagList::parse(value) { + TagList::Any => false, + TagList::Tags(tags) => tags.iter().any(is_signed_tag), + }); + if_range || if_match + } +} + +/// Whether `tag` is a strong entity tag core gave a signed representation. +fn is_signed_tag(tag: &EntityTag<'_>) -> bool { + !tag.weak + && tag + .opaque + .strip_prefix(SIGNED_TAG_PREFIX) + .is_some_and(|digest| { + digest.len() == 64 + && digest + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) + }) +} + +/// The strong entity tag of signed bytes whose SHA-256 is `digest`. +fn signed_entity_tag(digest: &[u8]) -> String { + format!("\"{SIGNED_TAG_PREFIX}{}\"", hex::encode(digest)) +} + +/// Whether a request plausibly asks for media: the destination a browser +/// names, else the first type it accepts, else the path's file extension. +fn request_looks_like_media(path: &str, headers: &HeaderMap) -> bool { + if let Some(destination) = headers + .get("sec-fetch-dest") + .and_then(|value| value.to_str().ok()) + { + let destination = destination.trim(); + return MEDIA_DESTINATIONS + .iter() + .any(|media| destination.eq_ignore_ascii_case(media)); + } + let accepted = header_text(headers, &header::ACCEPT) + .split(',') + .next() + .map(media_type_of) + .unwrap_or_default(); + if ["image/", "audio/", "video/", "font/"] + .iter() + .any(|prefix| accepted.starts_with(prefix)) + { + return true; + } + path.rsplit('/') + .next() + .and_then(|name| name.rsplit_once('.')) + .is_some_and(|(_, extension)| { + MEDIA_EXTENSIONS + .iter() + .any(|media| extension.eq_ignore_ascii_case(media)) + }) +} + +/// Seconds since the Unix epoch now, from the clock every runtime offers. +fn unix_seconds() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_secs()) +} + +/// An answer core meant to sign and could not, made safe to send unsigned: +/// without the origin's validators, which a later conditional request would +/// use to pin the unsigned bytes, and held only briefly and only by the +/// reader, so the next request can be signed. +fn unsigned_fallback(mut response: Response) -> Response { + crate::cache_policy::remove_owned_cache_headers(&mut response); + let headers = response.headers_mut(); + for name in ["etag", "last-modified", "expires"] { + headers.remove(name); + } + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_static(UNSIGNED_FALLBACK_CACHE_CONTROL), + ); + response +} + +/// Every line of header `name`, joined as one list. A value that is not +/// text stays present and matches nothing. +fn joined(headers: &HeaderMap, name: &HeaderName) -> Option { + let mut values = headers.get_all(name).iter().peekable(); + values.peek()?; + Some( + values + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + .collect::>() + .join(", "), + ) +} + +/// What names one signed representation before the origin is asked: the +/// reader's address, the origin core asks and the address it asks there. +#[derive(Debug, Clone, Copy)] +pub(crate) struct VariantIdentity<'a> { + /// `scheme://host/path` the reader asked for, without its query. + pub reader_url: &'a str, + /// What tells this origin apart from another at one address. + pub origin_identity: &'a str, + /// The address core fetches, exactly as fetched. + pub fetched_url: &'a str, + /// The reader's request carried no query, so + /// [`reader_url`](Self::reader_url) is the whole address it asked for. + pub reader_query_free: bool, +} + +/// Independent remaining browser and shared-cache lifetimes. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct OriginFreshness { + pub browser: Duration, + pub shared: Duration, +} + +/// What the origin's own headers said about its answer, read before core +/// changed or removed any of them, so a cache policy core writes never +/// stands in for the origin's. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct OriginFacts { + /// The origin marked the answer as one reader's, by a cookie, a + /// `Cache-Control` that forbids shared storage or a `Vary` on the + /// reader. + private: bool, + /// The origin forbade transforming the body. + no_transform: bool, + /// How long a signed representation of the answer may be kept. + keep_for: Option, + /// Freshness at receipt, before any response processing. + freshness: OriginFreshness, +} + +impl OriginFacts { + /// What `headers`, the origin's as it sent them, say. + pub(crate) fn of(headers: &HeaderMap) -> Self { + Self::of_at(headers, SystemTime::now()) + } + + /// Read against an explicit receipt time, also used by deterministic tests. + pub(crate) fn of_at(headers: &HeaderMap, received_at: SystemTime) -> Self { + let freshness = origin_freshness(headers, received_at); + Self { + private: headers.contains_key(header::SET_COOKIE) + || cache_control_forbids_shared_storage(headers) + || varies_by_reader(headers), + no_transform: cache_control_has(headers, "no-transform"), + keep_for: (!freshness.shared.is_zero()).then_some(freshness.shared), + freshness, + } + } + + /// The origin marked the answer as one reader's, by a cookie, a + /// `Cache-Control` that forbids shared storage or a `Vary` on the + /// reader. + pub(crate) fn is_private(&self) -> bool { + self.private + } + + /// The origin forbade transforming the body, by `no-transform`. + pub(crate) fn forbids_transform(&self) -> bool { + self.no_transform + } + + /// Freshness after processing, without borrowing the shared lifetime for + /// the browser or granting a new lifetime when origin dates are removed. + pub(crate) fn remaining_freshness(&self, elapsed: Duration) -> OriginFreshness { + OriginFreshness { + browser: self.freshness.browser.saturating_sub(elapsed), + shared: self.freshness.shared.saturating_sub(elapsed), + } + } +} + +/// Where the response handed to [`MediaClaim::deliver`] came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Answered { + /// The origin, to the request as core sent it, with what its own + /// headers said. + Origin(OriginFacts), + /// A kept signed representation [`MediaClaim::kept`] found. + Kept, +} + +/// What [`MediaClaim::deliver`] decided. +pub(crate) enum Delivery { + /// The response to send. + Send(Response), + /// The origin answered with a part, a `HEAD`, or a `412` or `416` a + /// reader's signed-length range or signed tag brings, for a + /// representation that may be signed. The caller asks the origin once + /// more for the whole representation, see + /// [`strip_representation_preconditions`], and hands the answer to + /// [`MediaClaim::deliver_whole`], with this response to send where that + /// fails. + FetchWhole(Response), + /// The origin answered with a source larger than the signer's bounded + /// limit that the large lane admits, pinned to its version. The caller + /// hands it to [`MediaClaim::deliver_large`] with how the origin is + /// asked again. + Large(LargeAnswer), +} + +/// A request the services' [`MediaSigner`] may sign the answer to. +pub(crate) struct MediaClaim { + signer: Arc, + reader_method: Method, + preconditions: ReaderPreconditions, + spec: CacheSpec, + /// The answer may be kept for, and a kept one served to, other + /// readers: the request carries nothing that could make the origin's + /// answer this reader's own. + shared: bool, + /// The request lets a kept signed representation answer it: it is + /// shared, does not ask for revalidation, and plausibly asks for media. + reads_kept: bool, + /// The request plausibly asks for media. + looks_like_media: bool, + /// The reader's `If-Range` or `If-Match` names a signed + /// representation, which the origin cannot judge. + names_signed_bytes: bool, + /// The answer is the platform image optimizer's, see + /// [`answered_by_optimizer`](Self::answered_by_optimizer). + optimizer_output: bool, + /// The claim answered the reader's preconditions against the + /// representation it delivered, see + /// [`answer_unsigned`](Self::answer_unsigned). + conditions_answered: AtomicBool, +} + +impl MediaClaim { + /// The claim the request's services' signer makes on a request, or + /// `None` where no signer is registered, the method is neither `GET` + /// nor `HEAD`, the request carries `Authorization`, or the signer leaves + /// `path` alone. + /// + /// `shared` says whether nothing the request carries beyond these + /// headers could make the origin's answer this reader's own, being a + /// cookie sent to the origin, a module's request state or a + /// personalized delivery. Only a shared request reads or writes a kept + /// signed representation. + pub(crate) fn of_request( + settings: &Settings, + services: &RuntimeServices, + method: &Method, + path: &str, + headers: &HeaderMap, + shared: bool, + ) -> Option { + let signer = services.media_signer()?; + let claimed = (*method == Method::GET || *method == Method::HEAD) + && !headers.contains_key(header::AUTHORIZATION) + && signer.handles_path(path); + if !claimed { + return None; + } + let preconditions = ReaderPreconditions::of(headers); + let looks_like_media = request_looks_like_media(path, headers); + let names_signed_bytes = preconditions.names_signed_bytes(); + Some(Self { + signer: Arc::clone(signer), + reader_method: method.clone(), + preconditions, + spec: CacheSpec { + name: ASSETS_CACHE, + lifetimes: settings + .cache + .assets + .lifetimes(crate::publisher::ASSET_DEFAULT_LIFETIMES), + memory_entries: KEPT_MEMORY_ENTRIES, + }, + shared, + reads_kept: shared + && looks_like_media + && !crate::publisher::request_asks_for_revalidation(headers), + looks_like_media, + names_signed_bytes, + optimizer_output: false, + conditions_answered: AtomicBool::new(false), + }) + } + + /// The same claim on a request the platform's image optimizer answers. + /// + /// The optimizer reads the origin itself, so core never holds its + /// input. Its answer is signed only where it carries a C2PA manifest + /// store, the optimizer's own record of the input it read and what it + /// did, and the signer must refuse unless that manifest is present and + /// intact, see [`MediaJob::optimizer_output`]. Its answer depends on + /// the transformation and the reader's `Accept`, neither of which names + /// a kept representation, so nothing is read from or written to the + /// kept representations, and only a whole `200` to a `GET` is signed: + /// a part or `HEAD` is never asked for again past the optimizer. + pub(crate) fn answered_by_optimizer(mut self) -> Self { + self.shared = false; + self.reads_kept = false; + self.optimizer_output = true; + self + } + + /// Whether the request plausibly asks for media, by the destination + /// the browser names, the type it accepts first, or the file extension. + /// The origin is then asked for an identity-coded body, the only coding + /// signed. + pub(crate) fn looks_like_media(&self) -> bool { + self.looks_like_media + } + + /// Whether the reader's `If-Range` or `If-Match` names a signed + /// representation. The origin cannot judge such a precondition, so the + /// caller asks it for the whole representation, see + /// [`strip_representation_preconditions`], and core answers the + /// reader's range and preconditions against the signed bytes. + pub(crate) fn names_signed_bytes(&self) -> bool { + self.names_signed_bytes + } + + /// Whether the answer is the platform image optimizer's, see + /// [`answered_by_optimizer`](Self::answered_by_optimizer). + pub(crate) fn signs_optimizer_output(&self) -> bool { + self.optimizer_output + } + + /// The signed representation kept for `identity` under the signer's + /// current namespace, as an origin answer to hand to + /// [`deliver`](Self::deliver) with [`Answered::Kept`], or `None`. Only + /// a shared request that plausibly asks for media and asks for no + /// revalidation is looked up, so the signer's namespace is not resolved + /// for any other. + pub(crate) async fn kept( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + ) -> Option> { + if !self.reads_kept || !services.loading_cache_for(ASSETS_CACHE).keeps_values() { + return None; + } + let namespace = match self.signer.namespace(services).await { + Ok(namespace) => namespace, + Err(failure) => { + log::debug!( + "media signing: nothing kept looked up signer={} reason={failure}", + self.signer.signer_id() + ); + return None; + } + }; + let key = self.store_key(services, &namespace, identity); + let bytes = services + .load_cached( + &self.spec, + &key, + Box::new(|| Box::pin(async { Err(LoadError::new(NOT_KEPT)) })), + ) + .await + .ok()?; + let kept = decode_kept(&bytes, unix_seconds()); + if kept.is_none() { + log::warn!( + "media signing: kept representation unreadable or altered signer={}", + self.signer.signer_id() + ); + services + .loading_cache_for(ASSETS_CACHE) + .remove(&services.clone().without_connection(), &self.spec, &key) + .await; + } + kept + } + + /// Answers the reader from `response`: a kept signed representation, or + /// the origin's answer, signed where the signer signs it. + pub(crate) async fn deliver( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + answered: Answered, + response: Response, + ) -> Delivery { + let facts = match answered { + Answered::Kept => return Delivery::Send(self.respond_kept(response)), + Answered::Origin(facts) => facts, + }; + if self.optimizer_output { + return Delivery::Send( + self.deliver_optimized(services, identity, &facts, response) + .await, + ); + } + let signer_id = self.signer.signer_id(); + let status = response.status(); + let limit = to_u64(self.signer.max_source_bytes()); + match status { + StatusCode::OK if self.reader_method != Method::HEAD => { + if let Some(reason) = self.unsignable_reason(response.headers(), &facts) { + log::debug!( + "media signing: outcome=unsigned signer={signer_id} reason={reason}" + ); + return Delivery::Send(response); + } + let total = declared_length(response.headers()); + if total.is_some_and(|total| total > limit) { + return self.large_or_unsigned(services, response, total, &facts); + } + let (parts, body) = response.into_parts(); + let source = match self.capture_within_limit(&parts.headers, body).await { + Ok(source) => source, + Err(body) => return Delivery::Send(Response::from_parts(parts, body)), + }; + if is_fragmented_stream_part(&source) { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=fragmented_stream_part" + ); + return Delivery::Send(Response::from_parts(parts, EdgeBody::Once(source))); + } + match self.sign(services, identity, &parts.headers, &source).await { + Ok(signed) => Delivery::Send( + self.respond_signed(services, identity, parts, signed, facts.keep_for) + .await, + ), + Err(()) => Delivery::Send(unsigned_fallback(Response::from_parts( + parts, + EdgeBody::Once(source), + ))), + } + } + StatusCode::OK | StatusCode::PARTIAL_CONTENT => { + if let Some(reason) = self.unsignable_reason(response.headers(), &facts) { + log::debug!( + "media signing: outcome=unsigned signer={signer_id} reason={reason}" + ); + return Delivery::Send(response); + } + let total = if status == StatusCode::PARTIAL_CONTENT { + content_range_total(response.headers()) + } else { + declared_length(response.headers()) + }; + let too_large = if status == StatusCode::PARTIAL_CONTENT { + total.is_none_or(|total| total > limit) + } else { + total.is_some_and(|total| total > limit) + }; + if too_large { + return self.large_or_unsigned(services, response, total, &facts); + } + // A part or a `HEAD` is signed only where the signed whole + // can be kept, so the reader's next range is answered from + // it and the origin is not read whole for every range. + if !self.retains(services, &facts) { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=part_not_kept" + ); + return Delivery::Send(response); + } + Delivery::FetchWhole(response) + } + StatusCode::PRECONDITION_FAILED | StatusCode::RANGE_NOT_SATISFIABLE + if self.preconditions.range.is_some() || self.preconditions.if_match.is_some() => + { + // The reader's range or `If-Match` may describe the signed + // representation, which is longer than the origin's and has + // a tag of its own, so the origin's refusal is not the + // answer until the signed bytes are asked. + let too_large = + content_range_total(response.headers()).is_some_and(|total| total > limit); + if facts.private || too_large { + return Delivery::Send(response); + } + Delivery::FetchWhole(response) + } + StatusCode::NOT_MODIFIED => Delivery::Send( + self.decorate_not_modified(services, identity, &facts, response) + .await, + ), + _ => Delivery::Send(response), + } + } + + /// Answers the reader from the platform image optimizer's answer: a + /// whole `200` to a `GET` that carries a C2PA manifest store is signed, + /// and every other answer goes as the optimizer made it. + async fn deliver_optimized( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + facts: &OriginFacts, + response: Response, + ) -> Response { + let signer_id = self.signer.signer_id(); + if response.status() != StatusCode::OK || self.reader_method != Method::GET { + return response; + } + if let Some(reason) = self.unsignable_reason(response.headers(), facts) { + log::debug!("media signing: outcome=unsigned signer={signer_id} reason={reason}"); + return response; + } + let (parts, body) = response.into_parts(); + let source = match self.capture_within_limit(&parts.headers, body).await { + Ok(source) => source, + Err(body) => return Response::from_parts(parts, body), + }; + // The optimizer writes a manifest only where its input carried one. + // Without it nothing records what the optimizer read or did, so the + // answer is not signed. + if !carries_c2pa_manifest_store(&source) { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=image_optimizer_output_without_c2pa" + ); + return Response::from_parts(parts, EdgeBody::Once(source)); + } + // The optimizer may choose its output format from the reader's + // `Accept`, and it removes the origin's `Vary`, so a cache past core + // is told the signed bytes depend on it. + let mut parts = parts; + let varies_by_accept = parts + .headers + .get_all(header::VARY) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .any(|name| name.trim() == "*" || name.trim().eq_ignore_ascii_case("accept")); + if !varies_by_accept { + parts + .headers + .append(header::VARY, HeaderValue::from_static("Accept")); + } + match self.sign(services, identity, &parts.headers, &source).await { + Ok(signed) => { + self.respond_signed(services, identity, parts, signed, None) + .await + } + Err(()) => unsigned_fallback(Response::from_parts(parts, EdgeBody::Once(source))), + } + } + + /// Answers the reader from the origin's answer to the whole, + /// unconditional request [`Delivery::FetchWhole`] asked for, or sends + /// `fallback` where it cannot be signed. + /// + /// `whole` is the origin's answer as it came, whose own headers decide + /// what is signed and how long it is kept. The signed response is + /// written on `fallback`'s headers, which carry what core wrote for + /// this reader. + pub(crate) async fn deliver_whole( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + whole: Response, + fallback: Response, + ) -> Response { + let signer_id = self.signer.signer_id(); + // The answer sent where the whole cannot be signed already declares + // a manifest of the origin's own, which core never competes with. + if carries_manifest_link(fallback.headers()) { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=origin_manifest_link" + ); + return fallback; + } + let facts = OriginFacts::of(whole.headers()); + if whole.status() != StatusCode::OK { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=whole_fetch_status_{}", + whole.status().as_u16() + ); + return unsigned_fallback(fallback); + } + if let Some(reason) = self.unsignable_reason(whole.headers(), &facts) { + log::info!("media signing: outcome=unsigned signer={signer_id} reason={reason}"); + return fallback; + } + let (whole_parts, whole_body) = whole.into_parts(); + let Ok(source) = self + .capture_within_limit(&whole_parts.headers, whole_body) + .await + else { + return fallback; + }; + // A part of a fragmented stream goes as the origin answered it. + if is_fragmented_stream_part(&source) { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=fragmented_stream_part" + ); + return fallback; + } + match self + .sign(services, identity, &whole_parts.headers, &source) + .await + { + Ok(signed) => { + let (parts, _) = fallback.into_parts(); + self.respond_signed(services, identity, parts, signed, facts.keep_for) + .await + } + Err(()) => unsigned_fallback(fallback), + } + } + + /// Whether a signed whole made for this request can be kept: the request + /// is shared, the origin gave its answer a shared freshness, and the + /// deployment's asset cache keeps values. + fn retains(&self, services: &RuntimeServices, facts: &OriginFacts) -> bool { + self.shared + && facts.keep_for.is_some() + && services.loading_cache_for(ASSETS_CACHE).keeps_values() + } + + /// The whole body of a signable `200`, or the body as it stands where + /// it is larger than the signer's limit or could not be read, so the + /// reader still receives every byte the origin sent. + async fn capture_within_limit( + &self, + headers: &HeaderMap, + body: EdgeBody, + ) -> Result { + let signer_id = self.signer.signer_id(); + let limit = self.signer.max_source_bytes(); + if declared_length(headers).is_some_and(|length| length > to_u64(limit)) { + log::info!("media signing: outcome=unsigned signer={signer_id} reason=exceeds_limit"); + return Err(body); + } + match capture(body, limit).await { + Captured::Whole(source) => Ok(source), + Captured::Partial(body) => { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=exceeds_limit_or_unreadable" + ); + Err(body) + } + } + } + + /// Why a response is not handed to the signer, or `None` when it is. + /// `facts` are the origin's own, read before core wrote its cache + /// policy on `headers`. + fn unsignable_reason(&self, headers: &HeaderMap, facts: &OriginFacts) -> Option<&'static str> { + if facts.private || headers.contains_key(header::SET_COOKIE) { + return Some("origin_private"); + } + // Provenance the origin already links to describes these bytes, so + // they are neither signed again nor given a competing manifest. + if carries_manifest_link(headers) { + return Some("origin_manifest_link"); + } + if facts.no_transform { + return Some("no_transform"); + } + let encoding = header_text(headers, &header::CONTENT_ENCODING).trim(); + if !encoding.is_empty() && !encoding.eq_ignore_ascii_case("identity") { + return Some("content_encoded"); + } + let media_type = media_type_of(header_text(headers, &header::CONTENT_TYPE)); + if media_type.is_empty() || !self.signer.handles_media_type(&media_type) { + return Some("media_type"); + } + None + } + + /// Asks the signer for the signed representation of `source`, logging + /// the outcome with no address, since a reader's or origin's address + /// may carry a credential. + async fn sign( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + headers: &HeaderMap, + source: &Bytes, + ) -> Result { + let signer_id = self.signer.signer_id(); + let source_sha256 = hex::encode(Sha256::digest(source)); + let origin_host = url::Url::parse(identity.fetched_url) + .ok() + .and_then(|url| url.host_str().map(str::to_ascii_lowercase)); + let job = MediaJob { + reader_url: identity.reader_url, + content_type: header_text(headers, &header::CONTENT_TYPE), + source, + source_sha256: &source_sha256, + // No address serves the optimizer's answer. + source_reference: if self.optimizer_output { + None + } else { + source_reference(identity.fetched_url) + }, + source_origin_host: origin_host.as_deref(), + optimizer_output: self.optimizer_output, + }; + let checked = match self.signer.sign(services, &job).await { + Ok(signed) if signed.bytes.is_empty() => Err(MediaSigningFailure { + reason: "empty_signed_body", + }), + Ok(signed) + if signed + .headers + .iter() + .any(|(name, _)| is_reserved_header(name)) => + { + Err(MediaSigningFailure { + reason: "reserved_header", + }) + } + Ok(signed) if signed.namespace.is_empty() => Err(MediaSigningFailure { + reason: "no_namespace", + }), + other => other, + }; + match checked { + Ok(signed) => { + log::info!( + "media signing: outcome=signed signer={signer_id} source_bytes={} signed_bytes={}", + source.len(), + signed.bytes.len() + ); + Ok(signed) + } + Err(failure) => { + log::warn!("media signing: outcome=unsigned signer={signer_id} reason={failure}"); + Err(()) + } + } + } + + /// The signed representation, kept for `keep_for` where this request is + /// shared, and answered against the reader's request. Ranges are offered + /// only on a representation that was kept, so a reader's later range is + /// answered from it and never from a part of the origin's own bytes. + async fn respond_signed( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + mut parts: http::response::Parts, + signed: SignedMedia, + keep_for: Option, + ) -> Response { + let origin_age = digits(header_text(&parts.headers, &header::AGE).trim()).unwrap_or(0); + for name in ORIGIN_REPRESENTATION_HEADERS { + parts.headers.remove(name); + } + let digest = Sha256::digest(&signed.bytes); + let etag = signed_entity_tag(digest.as_slice()); + let repr_digest = format!( + "sha-256=:{}:", + base64::engine::general_purpose::STANDARD.encode(digest.as_slice()) + ); + let mut representation = HeaderMap::new(); + representation.insert(header::CONTENT_TYPE, signed.content_type); + representation.insert( + header::ETAG, + HeaderValue::from_str(&etag).expect("should build a hex entity tag"), + ); + representation.insert( + HeaderName::from_static("repr-digest"), + HeaderValue::from_str(&repr_digest).expect("should build a base64 digest"), + ); + for name in KEPT_ORIGIN_HEADERS { + for value in parts.headers.get_all(name) { + representation.append(HeaderName::from_static(name), value.clone()); + } + } + for (name, value) in signed.headers { + if name == header::LINK { + representation.append(name, value); + } else { + representation.insert(name, value); + } + } + let kept = match keep_for.filter(|_| self.shared) { + Some(lifetime) => { + let value = encode_kept(&KeptRecord { + kept_at: unix_seconds(), + age: origin_age, + headers: &representation, + body: &signed.bytes, + }); + self.keep(services, identity, &signed.namespace, value, lifetime) + .await + } + None => false, + }; + for name in representation.keys() { + parts.headers.remove(name); + } + for (name, value) in &representation { + parts.headers.append(name.clone(), value.clone()); + } + self.answer(parts, signed.bytes, kept) + } + + /// A kept signed representation, answered against the reader's + /// request. + fn respond_kept(&self, response: Response) -> Response { + let (parts, body) = response.into_parts(); + let bytes = body.into_bytes().unwrap_or_default(); + self.answer(parts, bytes, true) + } + + /// Answers the reader's preconditions and range against the signed + /// `bytes`, whose `ETag` is in `parts`. + fn answer( + &self, + parts: http::response::Parts, + bytes: Bytes, + ranges_offered: bool, + ) -> Response { + let length = to_u64(bytes.len()); + let etag = parts + .headers + .get(header::ETAG) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let outcome = evaluate( + &self.reader_method, + &self.preconditions, + etag.as_deref(), + None, + length, + ); + self.conditions_answered.store(true, Ordering::Relaxed); + self.apply(parts, bytes, outcome, ranges_offered) + } + + /// `response`, a whole `200` this claim delivered, answered against the + /// reader's preconditions where the claim has not already answered them, + /// which it has for every signed representation it delivered. For a + /// request whose preconditions were not sent to the origin, so the + /// origin's whole answer is checked against them here, by the + /// representation's own `ETag` and `Last-Modified`. A range is not + /// applied, so a passing answer goes whole, as RFC 9110 allows. + pub(crate) fn answer_unsigned(&self, response: Response) -> Response { + if self.conditions_answered.load(Ordering::Relaxed) { + return response; + } + answer_whole_with(&self.reader_method, &self.preconditions, response) + } +} + +/// `response`, a whole `200`, answered against the preconditions of a +/// reader's request carrying `reader_headers`, by the representation's own +/// `ETag` and `Last-Modified`, see [`MediaClaim::answer_unsigned`]: `412` +/// or `304` with no body, or the response as it is. A range is not applied. +pub(crate) fn answer_whole( + method: &Method, + reader_headers: &HeaderMap, + response: Response, +) -> Response { + answer_whole_with(method, &ReaderPreconditions::of(reader_headers), response) +} + +fn answer_whole_with( + method: &Method, + preconditions: &ReaderPreconditions, + response: Response, +) -> Response { + if response.status() != StatusCode::OK { + return response; + } + let headers = response.headers(); + let etag = headers + .get(header::ETAG) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let last_modified = headers + .get(header::LAST_MODIFIED) + .and_then(|value| value.to_str().ok()) + .and_then(|value| httpdate::parse_http_date(value.trim()).ok()); + let whole = ReaderPreconditions { + range: None, + if_range: None, + ..preconditions.clone() + }; + let outcome = evaluate(method, &whole, etag.as_deref(), last_modified, 0); + let (mut parts, body) = response.into_parts(); + match outcome { + Outcome::NotModified => { + parts.status = StatusCode::NOT_MODIFIED; + for name in ["content-length", "content-range", "transfer-encoding"] { + parts.headers.remove(name); + } + Response::from_parts(parts, EdgeBody::empty()) + } + Outcome::PreconditionFailed => { + parts.status = StatusCode::PRECONDITION_FAILED; + for name in [ + "content-type", + "content-range", + "transfer-encoding", + "content-encoding", + ] { + parts.headers.remove(name); + } + parts + .headers + .insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); + Response::from_parts(parts, EdgeBody::empty()) + } + _ => Response::from_parts(parts, body), + } +} + +/// Removes the headers that describe the origin's bytes or one transfer of +/// them, for a representation core made of them. +pub(crate) fn remove_origin_representation_headers(headers: &mut HeaderMap) { + for name in ORIGIN_REPRESENTATION_HEADERS { + headers.remove(name); + } +} + +impl MediaClaim { + /// Keeps a signed representation for `identity` under `namespace`, the + /// one it was signed under, so a later request is answered before the + /// origin is asked. Whether it was kept. + async fn keep( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + namespace: &str, + value: Bytes, + lifetime: Duration, + ) -> bool { + if !services.loading_cache_for(ASSETS_CACHE).keeps_values() { + return false; + } + let key = self.store_key(services, namespace, identity); + let stored = services + .load_cached( + &self.spec, + &key, + Box::new(move || { + Box::pin(async move { Ok(LoadedValue::new(value).with_lifetime(lifetime)) }) + }), + ) + .await; + match stored { + Ok(_) => true, + Err(error) => { + log::debug!( + "media signing: representation not kept signer={} error={error}", + self.signer.signer_id() + ); + false + } + } + } + + /// The key a signed representation for `identity` is kept under, bound + /// to `namespace`. + fn store_key( + &self, + services: &RuntimeServices, + namespace: &str, + identity: &VariantIdentity<'_>, + ) -> StoreKey { + let mut digest = Sha256::new(); + for part in [ + self.signer.signer_id(), + namespace, + identity.reader_url, + identity.origin_identity, + identity.fetched_url, + ] { + digest.update(part.as_bytes()); + digest.update([0]); + } + StoreKey::new( + services.server_key_prefix(), + ASSETS_CACHE, + format!("media-{}", hex::encode(digest.finalize().as_slice())), + ) + } + + /// Writes `outcome` on the response: its status, its framing headers and + /// the part of `bytes` it carries. + fn apply( + &self, + mut parts: http::response::Parts, + bytes: Bytes, + outcome: Outcome, + ranges_offered: bool, + ) -> Response { + let length = to_u64(bytes.len()); + let headers = &mut parts.headers; + for name in [ + "content-range", + "content-length", + "transfer-encoding", + "content-encoding", + ] { + headers.remove(name); + } + headers.insert( + header::ACCEPT_RANGES, + HeaderValue::from_static(if ranges_offered { "bytes" } else { "none" }), + ); + let carries_body = self.reader_method != Method::HEAD; + let body = match outcome { + Outcome::Full => { + parts.status = StatusCode::OK; + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(length)); + if carries_body { + EdgeBody::Once(bytes) + } else { + EdgeBody::empty() + } + } + Outcome::NotModified => { + parts.status = StatusCode::NOT_MODIFIED; + EdgeBody::empty() + } + Outcome::PreconditionFailed => { + parts.status = StatusCode::PRECONDITION_FAILED; + headers.remove(header::CONTENT_TYPE); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); + EdgeBody::empty() + } + Outcome::Partial { first, last } => { + parts.status = StatusCode::PARTIAL_CONTENT; + headers.insert( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes {first}-{last}/{length}")) + .expect("should build a content range"), + ); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(last - first + 1)); + let from = usize::try_from(first) + .unwrap_or(bytes.len()) + .min(bytes.len()); + let to = usize::try_from(last) + .map_or(bytes.len(), |last| last.saturating_add(1)) + .min(bytes.len()); + EdgeBody::Once(bytes.slice(from..to.max(from))) + } + Outcome::Unsatisfiable => { + parts.status = StatusCode::RANGE_NOT_SATISFIABLE; + headers.insert( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes */{length}")) + .expect("should build a content range"), + ); + headers.remove(header::CONTENT_TYPE); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); + EdgeBody::empty() + } + }; + Response::from_parts(parts, body) + } +} + +/// Origin freshness at receipt. Revalidation-only answers and reader-varying +/// responses authorize no reusable copy. Shared freshness falls back to +/// `max-age` only when `s-maxage` is absent, never when it is malformed. +fn origin_freshness(headers: &HeaderMap, received_at: SystemTime) -> OriginFreshness { + if ["no-store", "no-cache", "private"] + .iter() + .any(|directive| cache_control_has(headers, directive)) + || headers + .get_all(header::VARY) + .iter() + .flat_map(|value| value.to_str().unwrap_or("*").split(',')) + .map(str::trim) + .any(|name| !name.is_empty() && !name.eq_ignore_ascii_case("accept-encoding")) + { + return OriginFreshness::default(); + } + let age = match headers.get(header::AGE) { + Some(value) => match value.to_str().ok().and_then(|value| digits(value.trim())) { + Some(seconds) => Duration::from_secs(seconds), + None => return OriginFreshness::default(), + }, + None => Duration::ZERO, + }; + let apparent_age = match headers.get(header::DATE) { + Some(value) => match value + .to_str() + .ok() + .and_then(|value| httpdate::parse_http_date(value.trim()).ok()) + { + Some(date) => received_at + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .saturating_sub( + date.duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(), + ), + None => return OriginFreshness::default(), + }, + None => Duration::ZERO, + }; + let current_age = age.max(apparent_age); + let browser = cache_control_seconds(headers, "max-age"); + let shared = match cache_control_seconds(headers, "s-maxage") { + Ok(None) => browser, + value => value, + }; + let remaining = |seconds: Result, ()>| { + Duration::from_secs(seconds.ok().flatten().unwrap_or(0)).saturating_sub(current_age) + }; + OriginFreshness { + browser: remaining(browser), + shared: remaining(shared), + } +} + +/// An absent directive permits fallback; an invalid or repeated one does not. +fn cache_control_seconds(headers: &HeaderMap, directive: &str) -> Result, ()> { + let mut seconds = None; + for line in headers.get_all(header::CACHE_CONTROL) { + for part in line.to_str().map_err(|_| ())?.split(',') { + let (name, value) = part.split_once('=').unwrap_or((part, "")); + if !name.trim().eq_ignore_ascii_case(directive) { + continue; + } + if seconds.is_some() { + return Err(()); + } + let value = value.trim(); + let value = if let Some(quoted) = value.strip_prefix('"') { + quoted.strip_suffix('"').ok_or(())? + } else { + value + }; + seconds = Some(digits(value).ok_or(())?); + } + } + Ok(seconds) +} + +/// What a kept signed representation holds besides its key. +struct KeptRecord<'a> { + /// When it was kept, in seconds since the Unix epoch. + kept_at: u64, + /// The `Age` the origin's answer carried when it was kept. + age: u64, + headers: &'a HeaderMap, + body: &'a Bytes, +} + +/// A kept signed representation, as written: [`KEPT_MAGIC`], when it was +/// kept and the age it had then, the header count, each header's name and +/// value with their lengths, then the body. +fn encode_kept(record: &KeptRecord<'_>) -> Bytes { + let mut out = BytesMut::with_capacity(record.body.len() + 512); + out.extend_from_slice(KEPT_MAGIC); + out.extend_from_slice(&record.kept_at.to_be_bytes()); + out.extend_from_slice(&record.age.to_be_bytes()); + let count = u32::try_from(record.headers.len()).unwrap_or(u32::MAX); + out.extend_from_slice(&count.to_be_bytes()); + for (name, value) in record + .headers + .iter() + .take(usize::try_from(count).unwrap_or(usize::MAX)) + { + let name = name.as_str().as_bytes(); + let value = value.as_bytes(); + out.extend_from_slice(&u32::try_from(name.len()).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(name); + out.extend_from_slice(&u32::try_from(value.len()).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(value); + } + out.extend_from_slice(record.body); + out.freeze() +} + +/// A kept signed representation as a `200` answer at `now`, with the `Age` +/// it has reached, or `None` where the bytes are not one [`encode_kept`] +/// wrote or the body is not the one its entity tag names. +fn decode_kept(bytes: &Bytes, now: u64) -> Option> { + let mut reader = KeptReader { + rest: bytes.strip_prefix(KEPT_MAGIC.as_slice())?, + }; + let kept_at = reader.number()?; + let age = reader.number()?; + let count = reader.length()?; + let mut headers = HeaderMap::new(); + for _ in 0..count { + let name_length = reader.length()?; + let name = HeaderName::from_bytes(reader.take(name_length)?).ok()?; + let value_length = reader.length()?; + let value = HeaderValue::from_bytes(reader.take(value_length)?).ok()?; + headers.append(name, value); + } + let body = bytes.slice(bytes.len() - reader.rest.len()..); + let expected = signed_entity_tag(Sha256::digest(&body).as_slice()); + if header_text(&headers, &header::ETAG) != expected { + return None; + } + headers.insert( + header::AGE, + HeaderValue::from(age.saturating_add(now.saturating_sub(kept_at))), + ); + let mut response = Response::new(EdgeBody::Once(body)); + *response.headers_mut() = headers; + Some(response) +} + +/// Reads a kept representation's fields in order. +struct KeptReader<'a> { + rest: &'a [u8], +} + +impl<'a> KeptReader<'a> { + fn take(&mut self, length: usize) -> Option<&'a [u8]> { + let taken = self.rest.get(..length)?; + self.rest = self.rest.get(length..)?; + Some(taken) + } + + fn length(&mut self) -> Option { + let bytes: [u8; 4] = self.take(4)?.try_into().ok()?; + usize::try_from(u32::from_be_bytes(bytes)).ok() + } + + fn number(&mut self) -> Option { + let bytes: [u8; 8] = self.take(8)?.try_into().ok()?; + Some(u64::from_be_bytes(bytes)) + } +} + +/// The fetched address a signer may name as the source, being the address +/// exactly as fetched when it carries no user information, query or +/// fragment, which could hold a credential or a reader's data. +fn source_reference(fetched_url: &str) -> Option<&str> { + let parsed = url::Url::parse(fetched_url).ok()?; + let plain = parsed.username().is_empty() + && parsed.password().is_none() + && parsed.query().is_none() + && parsed.fragment().is_none(); + plain.then_some(fetched_url) +} + +/// Whether `Vary` names `*`, `Cookie` or `Authorization`, so the origin's +/// answer may differ between readers. +fn varies_by_reader(headers: &HeaderMap) -> bool { + headers + .get_all(header::VARY) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .map(str::trim) + .any(|name| { + name == "*" + || name.eq_ignore_ascii_case("cookie") + || name.eq_ignore_ascii_case("authorization") + }) +} + +/// Whether the origin's own `Link` already names a C2PA manifest, so the +/// representation carries provenance core must keep as it came rather +/// than sign again or compete with. +fn carries_manifest_link(headers: &HeaderMap) -> bool { + headers + .get_all(header::LINK) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(';')) + .filter_map(|parameter| { + let (name, relations) = parameter.trim().split_once('=')?; + name.trim().eq_ignore_ascii_case("rel").then_some(relations) + }) + .flat_map(|relations| { + relations.split(|character: char| { + character.is_ascii_whitespace() || character == '"' || character == ',' + }) + }) + .any(|relation| relation.eq_ignore_ascii_case("c2pa-manifest")) +} + +/// The JUMBF description box of a C2PA manifest store: the `jumd` box type +/// followed by the C2PA manifest store UUID, +/// `63327061-0011-0010-8000-00AA00389B71`. Every embedding of a store, a +/// JPEG `APP11` segment, a PNG `caBX` chunk, a WebP `C2PA` chunk, an ISO +/// BMFF `uuid` box or a JPEG XL `jumb` box, carries these bytes +/// contiguously at the head of the store. +const C2PA_STORE_DESCRIPTION: [u8; 20] = [ + b'j', b'u', b'm', b'd', 0x63, 0x32, 0x70, 0x61, 0x00, 0x11, 0x00, 0x10, 0x80, 0x00, 0x00, 0xAA, + 0x00, 0x38, 0x9B, 0x71, +]; + +/// Whether `bytes` carry a C2PA manifest store. This is presence only: +/// whether the store is well placed and intact is the signer's to check, +/// see [`MediaJob::optimizer_output`]. +fn carries_c2pa_manifest_store(bytes: &[u8]) -> bool { + bytes + .windows(C2PA_STORE_DESCRIPTION.len()) + .any(|window| window == C2PA_STORE_DESCRIPTION) +} + +/// The most boxes [`bmff_boxes`] reads at one level before it gives up. +const MAX_BMFF_BOXES: usize = 4096; + +/// The type and body range of each ISO BMFF box laid end to end in +/// `bytes`, or `None` where they are not well formed, or are more than +/// [`MAX_BMFF_BOXES`]. A size of `1` reads a 64-bit size after the type, +/// and a size of `0` runs the box to the end of `bytes`. +fn bmff_boxes(bytes: &[u8]) -> Option)>> { + let mut boxes = Vec::new(); + let mut offset = 0_usize; + while offset < bytes.len() { + if boxes.len() == MAX_BMFF_BOXES { + return None; + } + let header = bytes.get(offset..offset.checked_add(8)?)?; + let size32 = u32::from_be_bytes(header.get(0..4)?.try_into().ok()?); + let kind: [u8; 4] = header.get(4..8)?.try_into().ok()?; + let (header_len, size) = match size32 { + 0 => (8, bytes.len() - offset), + 1 => { + let large = bytes.get(offset + 8..offset.checked_add(16)?)?; + let size = u64::from_be_bytes(large.try_into().ok()?); + (16, usize::try_from(size).ok()?) + } + size => (8, usize::try_from(size).ok()?), + }; + if size < header_len { + return None; + } + let end = offset.checked_add(size)?; + if end > bytes.len() { + return None; + } + boxes.push((kind, offset + header_len..end)); + offset = end; + } + Some(boxes) +} + +/// Whether `bytes` are a part of a fragmented ISO BMFF stream, which is +/// never signed alone: its provenance, such as C2PA verifiable segment +/// info, binds the stream's init segment and media segments together, so +/// a manifest added to one part would compete with the stream's. +/// +/// A media segment starts with `styp`, or has a `moof` before its first +/// `mdat`, and has no `moov`. An init segment starts with `ftyp`, has a +/// `moov` that holds an `mvex`, and has no `mdat`. A progressive file, +/// whose `moov` holds no `mvex` or which carries an `mdat` beside its +/// `moov`, is not a part, and neither is anything not well formed. +fn is_fragmented_stream_part(bytes: &[u8]) -> bool { + let Some(boxes) = bmff_boxes(bytes) else { + return false; + }; + let Some((first, _)) = boxes.first() else { + return false; + }; + let moov = boxes.iter().find(|(kind, _)| kind == b"moov"); + let first_mdat = boxes.iter().position(|(kind, _)| kind == b"mdat"); + let moof_before_mdat = boxes + .iter() + .take(first_mdat.unwrap_or(boxes.len())) + .any(|(kind, _)| kind == b"moof"); + let media_segment = (first == b"styp" || moof_before_mdat) && moov.is_none(); + let init_segment = first == b"ftyp" + && first_mdat.is_none() + && moov.is_some_and(|(_, body)| { + bytes + .get(body.clone()) + .and_then(bmff_boxes) + .is_some_and(|children| children.iter().any(|(kind, _)| kind == b"mvex")) + }); + media_segment || init_segment +} + +/// Whether a `Cache-Control` line names `directive`. +fn cache_control_has(headers: &HeaderMap, directive: &str) -> bool { + headers + .get_all(header::CACHE_CONTROL) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .map(|part| part.split('=').next().unwrap_or_default().trim()) + .any(|name| name.eq_ignore_ascii_case(directive)) +} + +fn header_text<'a>(headers: &'a HeaderMap, name: &HeaderName) -> &'a str { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() +} + +/// The origin's `Content-Length`, where it sent one that parses. +fn declared_length(headers: &HeaderMap) -> Option { + digits(header_text(headers, &header::CONTENT_LENGTH).trim()) +} + +/// The complete length a `206`'s `Content-Range: bytes a-b/total` names. +fn content_range_total(headers: &HeaderMap) -> Option { + let value = header_text(headers, &header::CONTENT_RANGE).trim(); + let (unit, range) = value.split_once(' ')?; + if !unit.eq_ignore_ascii_case("bytes") { + return None; + } + digits(range.rsplit_once('/')?.1.trim()) +} + +fn to_u64(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +/// A decimal number of ASCII digits alone, which `str::parse` would also +/// take with a leading `+`. +fn digits(value: &str) -> Option { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + value.parse().ok() +} + +/// A body read up to a limit. +enum Captured { + /// The whole body, within the limit. + Whole(Bytes), + /// A body over the limit, or one whose stream failed, rebuilt from what + /// was read and what was not, so the reader receives what it would have. + Partial(EdgeBody), +} + +/// Reads `body` whole when it is no larger than `limit`. +async fn capture(body: EdgeBody, limit: usize) -> Captured { + match body { + EdgeBody::Once(bytes) if bytes.len() <= limit => Captured::Whole(bytes), + once @ EdgeBody::Once(_) => Captured::Partial(once), + EdgeBody::Stream(mut stream) => { + let mut chunks = Vec::new(); + let mut total = 0_usize; + while let Some(item) = stream.next().await { + match item { + Ok(chunk) => { + total = total.saturating_add(chunk.len()); + chunks.push(chunk); + if total > limit { + let read = futures::stream::iter(chunks.into_iter().map(Ok)); + return Captured::Partial(EdgeBody::Stream( + read.chain(stream).boxed_local(), + )); + } + } + Err(error) => { + let read = chunks + .into_iter() + .map(Ok) + .chain(std::iter::once(Err(error))); + return Captured::Partial(EdgeBody::Stream( + futures::stream::iter(read).boxed_local(), + )); + } + } + } + let mut whole = BytesMut::with_capacity(total); + for chunk in chunks { + whole.extend_from_slice(&chunk); + } + Captured::Whole(whole.freeze()) + } + } +} + +/// How a request is answered from a signed representation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Outcome { + /// `200` with the whole representation. + Full, + /// `304`. + NotModified, + /// `412`. + PreconditionFailed, + /// `206` with bytes `first` to `last`, inclusive. + Partial { first: u64, last: u64 }, + /// `416`. + Unsatisfiable, +} + +/// Evaluates the reader's preconditions and range against a representation +/// in the order RFC 9110 section 13.2.2 gives. `etag` and `last_modified` +/// are the representation's own validators. A signed representation has no +/// modification date, so it passes `None` and `If-Modified-Since` and +/// `If-Unmodified-Since` are ignored; an `If-Range` date never holds. A +/// `Range` is honored on a `GET` alone, for one range. Several ranges are +/// answered with the whole representation, which the RFC allows. +fn evaluate( + method: &Method, + preconditions: &ReaderPreconditions, + etag: Option<&str>, + last_modified: Option, + length: u64, +) -> Outcome { + let current = etag.and_then(single_entity_tag); + let date_of = + |value: Option<&str>| value.and_then(|value| httpdate::parse_http_date(value.trim()).ok()); + + if let Some(value) = preconditions.if_match.as_deref() { + let matched = match TagList::parse(value) { + TagList::Any => true, + TagList::Tags(tags) => current + .as_ref() + .is_some_and(|current| tags.iter().any(|tag| tag.strong_eq(current))), + }; + if !matched { + return Outcome::PreconditionFailed; + } + } else if let (Some(since), Some(modified)) = ( + date_of(preconditions.if_unmodified_since.as_deref()), + last_modified, + ) && modified > since + { + return Outcome::PreconditionFailed; + } + + if let Some(value) = preconditions.if_none_match.as_deref() { + let matched = match TagList::parse(value) { + TagList::Any => true, + TagList::Tags(tags) => current + .as_ref() + .is_some_and(|current| tags.iter().any(|tag| tag.opaque == current.opaque)), + }; + if matched { + return if *method == Method::GET || *method == Method::HEAD { + Outcome::NotModified + } else { + Outcome::PreconditionFailed + }; + } + } else if (*method == Method::GET || *method == Method::HEAD) + && let (Some(since), Some(modified)) = ( + date_of(preconditions.if_modified_since.as_deref()), + last_modified, + ) + && modified <= since + { + return Outcome::NotModified; + } + + if *method != Method::GET { + return Outcome::Full; + } + let Some(range) = preconditions.range.as_deref() else { + return Outcome::Full; + }; + if let Some(if_range) = preconditions.if_range.as_deref() { + let holds = single_entity_tag(if_range) + .zip(current.as_ref()) + .is_some_and(|(tag, current)| tag.strong_eq(current)); + if !holds { + return Outcome::Full; + } + } + match parse_single_range(range, length) { + RangeRequest::Ignore => Outcome::Full, + RangeRequest::Satisfiable { first, last } => Outcome::Partial { first, last }, + RangeRequest::Unsatisfiable => Outcome::Unsatisfiable, + } +} + +/// One entity tag. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct EntityTag<'a> { + weak: bool, + opaque: &'a str, +} + +impl EntityTag<'_> { + fn strong_eq(&self, other: &EntityTag<'_>) -> bool { + !self.weak && !other.weak && self.opaque == other.opaque + } +} + +/// The entity tag at the start of `input` and what follows it. +fn split_entity_tag(input: &str) -> Option<(EntityTag<'_>, &str)> { + let (weak, quoted) = match input.strip_prefix("W/") { + Some(rest) => (true, rest), + None => (false, input), + }; + let body = quoted.strip_prefix('"')?; + let end = body.find('"')?; + Some(( + EntityTag { + weak, + opaque: &body[..end], + }, + &body[end + 1..], + )) +} + +/// A header holding exactly one entity tag, such as `ETag`. +fn single_entity_tag(value: &str) -> Option> { + let (tag, rest) = split_entity_tag(value.trim())?; + rest.trim().is_empty().then_some(tag) +} + +/// An `If-Match` or `If-None-Match` value. +enum TagList<'a> { + Any, + Tags(Vec>), +} + +impl<'a> TagList<'a> { + /// The tags of a list, as far as it parses. A tag may hold a comma, so + /// the list is read tag by tag. + fn parse(value: &'a str) -> Self { + if value.trim() == "*" { + return Self::Any; + } + let mut tags = Vec::new(); + let mut rest = value; + loop { + rest = rest.trim_start_matches([',', ' ', '\t']); + if rest.is_empty() { + break; + } + let Some((tag, after)) = split_entity_tag(rest) else { + break; + }; + tags.push(tag); + rest = after; + } + Self::Tags(tags) + } +} + +/// What a `Range` header asks of a representation of known length. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RangeRequest { + /// Not one byte range this answers, so the whole representation goes. + Ignore, + Satisfiable { + first: u64, + last: u64, + }, + Unsatisfiable, +} + +fn parse_single_range(value: &str, length: u64) -> RangeRequest { + let Some((unit, set)) = value.trim().split_once('=') else { + return RangeRequest::Ignore; + }; + if !unit.trim().eq_ignore_ascii_case("bytes") { + return RangeRequest::Ignore; + } + let mut specs = set + .split(',') + .map(str::trim) + .filter(|spec| !spec.is_empty()); + let (Some(spec), None) = (specs.next(), specs.next()) else { + return RangeRequest::Ignore; + }; + let Some((first, last)) = spec.split_once('-') else { + return RangeRequest::Ignore; + }; + let (first, last) = (first.trim(), last.trim()); + if first.is_empty() { + let Some(suffix) = digits(last) else { + return RangeRequest::Ignore; + }; + if suffix == 0 || length == 0 { + return RangeRequest::Unsatisfiable; + } + return RangeRequest::Satisfiable { + first: length.saturating_sub(suffix), + last: length - 1, + }; + } + let Some(first) = digits(first) else { + return RangeRequest::Ignore; + }; + let last = if last.is_empty() { + None + } else { + match digits(last) { + Some(last) if last >= first => Some(last), + _ => return RangeRequest::Ignore, + } + }; + if first >= length { + return RangeRequest::Unsatisfiable; + } + RangeRequest::Satisfiable { + first, + last: last.map_or(length - 1, |last| last.min(length - 1)), + } +} + +#[cfg(test)] +pub(crate) mod test_support; + +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/media_signing/large.rs b/crates/trusted-server-core/src/media_signing/large.rs new file mode 100644 index 0000000000..07e2ae355e --- /dev/null +++ b/crates/trusted-server-core/src/media_signing/large.rs @@ -0,0 +1,1582 @@ +//! The large-media lane: a representation larger than a signer's bounded +//! limit, signed without core ever holding it whole. +//! +//! The lane signs an unchanged source only. The bytes the reader receives +//! are the origin's own, so the origin's representation keeps its own +//! validators, length and range handling, and the signer only adds headers, +//! such as a manifest `Link`. Core adds a `Repr-Digest` of the whole source. +//! +//! Core admits an origin answer to the lane only where it can pin one +//! version of the source and keep the outcome: a shared request, an +//! identity-coded answer the origin keeps shareable for a while and lets +//! be transformed, a cache that keeps values, a single strong `ETag`, a +//! known total length above the signer's bounded limit and within its +//! large limit, a platform that streams response bodies and offers a +//! timer, and no C2PA manifest `Link` of the origin's own. Before anything +//! is hashed, core also requires a public address to name the source by: +//! the fetched address where it is plain `https` and core does not sign +//! the request for it, or else the reader's query-free `https` address. +//! An answer that fails any of these goes to the reader as the origin sent +//! it, validators and all; an answer with no strong `ETag` is never given +//! an invented version. Only a reader's `GET` of an answer whose origin +//! says `Accept-Ranges: bytes` starts the hashing of a version; a `HEAD`, +//! or a `GET` without that, is answered from a kept outcome or goes as it +//! came. +//! +//! Every later read of a version being signed, by core or the signer, asks +//! the origin through the route's own backend and origin signing for +//! exactly one byte range with `If-Match` on the pinned tag, and refuses an +//! answer whose status, `Content-Range`, `Content-Length`, coding or `ETag` +//! does not describe exactly that range of exactly that version. A changed +//! version therefore fails the read, never mixes into a digest or a signed +//! delivery. A version the lane declined has no signature to protect, so +//! where the origin's first answer can no longer be sent its reader gets +//! one fresh origin answer to the request as core first sent it, held by +//! no cache. +//! +//! What core keeps for the lane is the outcome only, never the bytes: for +//! a signed version the whole-source digest, the length and the signer's +//! headers, for a declined one (refused, failed or out of time) that it was +//! declined, briefly. Both are kept under the publisher's prefix, the +//! signer's current [`namespace`](super::MediaSigner::namespace), the +//! reader's address, the origin and the pinned tag, for no longer than the +//! origin's shared freshness. A changed or revoked policy names another +//! namespace, so it never finds an outcome kept under the old one. The +//! hashing and signing run as one load of the loading cache, which runs a +//! key's load once for every caller asking for it at the same time. + +use std::fmt; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use base64::Engine as _; +use bytes::{Bytes, BytesMut}; +use edgezero_core::body::Body as EdgeBody; +use futures::StreamExt as _; +use futures::future::Either; +use http::{HeaderMap, HeaderName, HeaderValue, Method, Request, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; +use web_time::Instant; + +use super::{ + Delivery, KeptReader, MediaClaim, MediaSigner, MediaSigningFailure, OriginFacts, Outcome, + VariantIdentity, carries_manifest_link, digits, evaluate, header_text, is_reserved_header, + single_entity_tag, source_reference, strip_representation_preconditions, to_u64, +}; +use crate::middleware::ResponseHeader; +use crate::origin_auth::OriginSigning; +use crate::platform::{ + LoadError, LoadedValue, Loader, PlatformHttpRequest, RuntimeServices, StoreKey, +}; +use crate::settings::ASSETS_CACHE; + +/// What [`MediaSigner::sign_large`](super::MediaSigner::sign_large) returns. +pub type LargeMediaSigning<'a> = + Pin> + 'a>>; + +/// One large, unchanged representation a signer is asked to sign. +#[derive(Debug, Clone, Copy)] +pub struct LargeMediaJob<'a> { + /// The address the reader asked for, `scheme://host/path`, without its + /// query. + pub reader_url: &'a str, + /// The origin's `Content-Type`, as it sent it. + pub content_type: &'a str, + /// The pinned version of the source, read at offsets. + pub source: &'a VersionedMediaSource<'a>, + /// SHA-256 of every byte of the pinned version, lower case hex, which + /// core computed in bounded chunks and checked against its length. The + /// bytes the reader receives are these bytes. + pub source_sha256: &'a str, + /// The address core fetched the source from, exactly as fetched, where + /// it is `https` and carries no user information, query or fragment. + pub source_reference: Option<&'a str>, + /// The reader's address, where it is `https` and the reader's request + /// carried no query. It names the source only because this lane + /// delivers the source unchanged. + pub reader_reference: Option<&'a str>, + /// The lower case host of the address core fetched the source from, + /// with no scheme, port, path, query or user information. + pub source_origin_host: &'a str, + /// What is left of the signer's + /// [`max_large_media_duration`](super::MediaSigner::max_large_media_duration) + /// after core's hashing. Core drops the signing when it is spent. + pub remaining: Duration, +} + +/// What a signer returns for a large representation: headers only, since +/// the bytes are the origin's. +#[derive(Debug, Clone)] +pub struct LargeSignedMedia { + /// Headers written on every answer of the signed version: `200`, `206`, + /// `HEAD` and `304`. A `Link` is added to the origin's own links, and + /// any other replaces the origin's header of that name. A name core + /// owns, see [`is_reserved_header`], fails the signing. + pub headers: Vec, + /// The policy and credential context the source was signed under, as + /// [`MediaSigner::namespace`](super::MediaSigner::namespace) gives it + /// while that context is current. Core keeps the metadata under it. + pub namespace: String, +} + +/// One version of a source: its strong entity tag and its length. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceVersion { + /// The strong tag, quoted, as `"opaque"`. + etag: String, + length: u64, +} + +impl SourceVersion { + /// The version a single strong entity tag `etag` and a length name, or + /// `None` where `etag` is weak, empty or not one tag. + #[must_use] + pub fn new(etag: &str, length: u64) -> Option { + let tag = single_entity_tag(etag)?; + (!tag.weak && !tag.opaque.is_empty()).then(|| Self { + etag: format!("\"{}\"", tag.opaque), + length, + }) + } + + /// The version an origin answer's headers name, where they carry + /// exactly one strong `ETag`. + fn of(headers: &HeaderMap, length: u64) -> Option { + let mut tags = headers.get_all(header::ETAG).iter(); + let (Some(tag), None) = (tags.next(), tags.next()) else { + return None; + }; + Self::new(tag.to_str().ok()?, length) + } + + /// The strong entity tag, quoted. + #[must_use] + pub fn etag(&self) -> &str { + &self.etag + } + + /// The length in bytes. + #[must_use] + pub fn length(&self) -> u64 { + self.length + } + + /// Whether an answer's headers carry this version's tag, strongly. + fn matches(&self, headers: &HeaderMap) -> bool { + let mut tags = headers.get_all(header::ETAG).iter(); + let (Some(tag), None) = (tags.next(), tags.next()) else { + return false; + }; + tag.to_str() + .ok() + .and_then(single_entity_tag) + .is_some_and(|tag| { + !tag.weak && self.etag.get(1..self.etag.len() - 1) == Some(tag.opaque) + }) + } +} + +/// Why a read of a [`VersionedMediaSource`] failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum MediaSourceError { + /// The origin no longer serves the pinned version: it answered `412` to + /// `If-Match`, or with a missing, weak or different `ETag`. Nothing + /// read from this source may be published. + #[display("the origin no longer serves the pinned version")] + VersionChanged, + /// The read reaches past the pinned version's length. + #[display("the read reaches past the pinned version")] + OutOfBounds, + /// The origin answered a pinned read with something other than exactly + /// the bytes asked for. + #[display("the origin answered a pinned read wrongly: {reason}")] + InvalidAnswer { + /// A short code naming what was wrong. + reason: &'static str, + }, + /// The read could not be sent, or its body failed or ended early. + #[display("a pinned read could not be completed")] + Transport, +} + +impl core::error::Error for MediaSourceError {} + +impl MediaSourceError { + /// A short code naming the failure, for logs and refusals. + #[must_use] + pub fn code(&self) -> &'static str { + match self { + Self::VersionChanged => "source_version_changed", + Self::OutOfBounds => "source_read_out_of_bounds", + Self::InvalidAnswer { reason } => reason, + Self::Transport => "source_transport", + } + } +} + +/// How core asks the origin for one more byte range of a source: the +/// address, the request's headers as they stood before it was signed, the +/// backend and the origin's signing. +#[derive(Clone, Copy)] +pub(crate) struct PinnedOrigin<'a> { + /// The address, exactly as fetched and signed. + pub(crate) url: &'a url::Url, + /// The origin request's headers before signing. + pub(crate) headers: &'a HeaderMap, + /// The backend the origin request went to. + pub(crate) backend_name: &'a str, + /// The origin's signing, applied fresh to every read. + pub(crate) auth: Option<&'a OriginSigning>, + /// Headers a signed read must not carry, removed before core writes its + /// own conditions and signs. + pub(crate) unsigned_headers: &'a [HeaderName], + /// Publisher URL mapping for a fresh unsigned response; asset proxies keep + /// the origin's navigation targets as on their ordinary response path. + pub(crate) response_header_rewrite: + Option>, +} + +/// One pinned version of a source, read at offsets from the origin. +/// +/// Every read asks the origin for exactly one byte range with `If-Match` +/// on the pinned tag, through the route's own backend and origin signing, +/// and fails unless the answer is exactly that range of that version. +pub struct VersionedMediaSource<'a> { + services: &'a RuntimeServices, + origin: PinnedOrigin<'a>, + version: SourceVersion, +} + +impl fmt::Debug for VersionedMediaSource<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("VersionedMediaSource") + .field("version", &self.version) + .finish_non_exhaustive() + } +} + +impl<'a> VersionedMediaSource<'a> { + /// A source read from `url` through `backend_name` with `headers`, an + /// origin that signs no request, pinned to `version`. + #[must_use] + pub fn new( + services: &'a RuntimeServices, + url: &'a url::Url, + headers: &'a HeaderMap, + backend_name: &'a str, + version: SourceVersion, + ) -> Self { + Self { + services, + origin: PinnedOrigin { + url, + headers, + backend_name, + auth: None, + unsigned_headers: &[], + response_header_rewrite: None, + }, + version, + } + } + + /// The version every read is pinned to. + #[must_use] + pub fn version(&self) -> &SourceVersion { + &self.version + } + + /// The pinned version's length in bytes. + #[must_use] + pub fn size(&self) -> u64 { + self.version.length + } + + /// Fills `destination` with the bytes at `offset` of the pinned version. + /// + /// # Errors + /// + /// [`MediaSourceError::OutOfBounds`] where the read reaches past the + /// version, [`MediaSourceError::VersionChanged`] where the origin no + /// longer serves it, and the other variants where the origin's answer + /// is not exactly the range asked for or does not arrive whole. + /// + /// # Performance + /// + /// One origin request per call, streamed into `destination`. + pub async fn read_exact_at( + &self, + offset: u64, + destination: &mut [u8], + ) -> Result<(), MediaSourceError> { + let length = to_u64(destination.len()); + let mut filled = 0_usize; + let mut visit = |chunk: &[u8]| { + let end = filled.saturating_add(chunk.len()); + if let Some(slot) = destination.get_mut(filled..end) { + slot.copy_from_slice(chunk); + } + filled = end; + }; + self.stream_range(offset, length, &mut visit).await + } + + /// Hands `visit` the bytes from `offset` for `length` bytes of the + /// pinned version, in order, in the chunks the origin sends. + /// + /// # Errors + /// + /// As [`read_exact_at`](Self::read_exact_at). `visit` may have seen + /// some of the bytes before a read fails. + /// + /// # Performance + /// + /// One origin request; nothing is held beyond one chunk. + pub async fn for_each_chunk( + &self, + offset: u64, + length: u64, + mut visit: F, + ) -> Result<(), MediaSourceError> { + self.stream_range(offset, length, &mut visit).await + } + + /// Whether the origin still serves the pinned version, by a pinned + /// read of its first byte. A signer asks this last before it publishes. + /// + /// # Errors + /// + /// As [`read_exact_at`](Self::read_exact_at). + pub async fn confirm_current(&self) -> Result<(), MediaSourceError> { + let mut first = [0_u8; 1]; + self.read_exact_at(0, &mut first).await + } + + /// The same source read through `origin`, which may sign its reads. + pub(crate) fn pinned( + services: &'a RuntimeServices, + origin: PinnedOrigin<'a>, + version: SourceVersion, + ) -> Self { + Self { + services, + origin, + version, + } + } + + /// Streams `length` bytes from `offset` to `visit`. + async fn stream_range( + &self, + offset: u64, + length: u64, + visit: &mut F, + ) -> Result<(), MediaSourceError> { + if length == 0 { + return Ok(()); + } + let last = offset + .checked_add(length - 1) + .filter(|last| *last < self.version.length) + .ok_or(MediaSourceError::OutOfBounds)?; + let body = self.send_pinned(offset, last).await?; + drain(body, length, visit).await + } + + /// Asks the origin for bytes `first` to `last` of the pinned version + /// and returns the body of an answer that is exactly that. + async fn send_pinned(&self, first: u64, last: u64) -> Result { + let request = self.pinned_request(first, last)?; + // The platform's error may name the address, which can carry a + // credential, so it is not logged. + let answer = self + .services + .http_client() + .send(request) + .await + .map_err(|_| MediaSourceError::Transport)?; + let (parts, body) = answer.response.into_parts(); + check_pinned(&self.version, parts.status, &parts.headers, first, last)?; + Ok(body) + } + + /// The origin request for bytes `first` to `last`: the request as it + /// stood before signing, with every condition the reader sent removed, + /// and only core's own `If-Match`, `Range` and identity coding written + /// before it is signed afresh. + fn pinned_request( + &self, + first: u64, + last: u64, + ) -> Result { + let mut headers = self.origin.base_headers(); + strip_representation_preconditions(&mut headers); + headers.insert( + header::ACCEPT_ENCODING, + HeaderValue::from_static("identity"), + ); + headers.insert( + header::IF_MATCH, + HeaderValue::from_str(&self.version.etag).map_err(|_| { + MediaSourceError::InvalidAnswer { + reason: "etag_not_a_header", + } + })?, + ); + headers.insert( + header::RANGE, + HeaderValue::from_str(&format!("bytes={first}-{last}")) + .expect("should build a byte range"), + ); + let request = self + .origin + .signed_request(&Method::GET, headers) + .map_err(|()| MediaSourceError::Transport)?; + Ok( + if self.services.http_client().supports_streaming_responses() { + request.with_stream_response() + } else { + request.with_max_response_body_bytes( + usize::try_from(last - first + 1).unwrap_or(usize::MAX), + ) + }, + ) + } +} + +impl PinnedOrigin<'_> { + /// The origin request's headers as they stood before signing, less + /// the headers a signed request must not carry where core signs it. + fn base_headers(&self) -> HeaderMap { + let mut headers = self.headers.clone(); + if self.auth.is_some() { + for name in self.unsigned_headers { + headers.remove(name); + } + } + headers + } + + /// A `method` request to the origin carrying `headers`, signed afresh + /// where the origin signs requests. Nothing is written after the + /// signature. + fn signed_request( + &self, + method: &Method, + mut headers: HeaderMap, + ) -> Result { + if let Some(auth) = self.auth { + auth.remove_namespace_headers(&mut headers); + auth.sign(method, self.url, &mut headers).map_err(|_| ())?; + } + let mut request = Request::builder() + .method(method.clone()) + .uri(self.url.as_str()) + .body(EdgeBody::empty()) + .map_err(|_| ())?; + *request.headers_mut() = headers; + Ok(PlatformHttpRequest::new(request, self.backend_name)) + } +} + +/// Checks that an answer to a pinned read of bytes `first` to `last` is +/// exactly that range of `version`. +fn check_pinned( + version: &SourceVersion, + status: StatusCode, + headers: &HeaderMap, + first: u64, + last: u64, +) -> Result<(), MediaSourceError> { + if status == StatusCode::PRECONDITION_FAILED { + return Err(MediaSourceError::VersionChanged); + } + if status != StatusCode::OK && status != StatusCode::PARTIAL_CONTENT { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_status", + }); + } + if !version.matches(headers) { + return Err(MediaSourceError::VersionChanged); + } + let encoding = header_text(headers, &header::CONTENT_ENCODING).trim(); + if !encoding.is_empty() && !encoding.eq_ignore_ascii_case("identity") { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_content_encoded", + }); + } + let whole = first == 0 && last.checked_add(1) == Some(version.length); + if status == StatusCode::PARTIAL_CONTENT { + if content_range(headers) != Some((first, last, version.length)) { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_content_range", + }); + } + } else if !whole { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_status", + }); + } + let declared = header_text(headers, &header::CONTENT_LENGTH).trim(); + if !declared.is_empty() && digits(declared) != Some(last - first + 1) { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_content_length", + }); + } + Ok(()) +} + +/// The first byte, last byte and complete length a `206`'s +/// `Content-Range: bytes first-last/length` names. +fn content_range(headers: &HeaderMap) -> Option<(u64, u64, u64)> { + let value = header_text(headers, &header::CONTENT_RANGE).trim(); + let (unit, range) = value.split_once(' ')?; + if !unit.eq_ignore_ascii_case("bytes") { + return None; + } + let (span, total) = range.trim().split_once('/')?; + let (first, last) = span.split_once('-')?; + Some(( + digits(first.trim())?, + digits(last.trim())?, + digits(total.trim())?, + )) +} + +/// Hands `visit` every chunk of `body`, which must hold exactly `expected` +/// bytes. A chunk that would pass `expected` is refused before `visit` +/// sees it. A stream that fails or ends early is a +/// [`MediaSourceError::Transport`]. +async fn drain( + body: EdgeBody, + expected: u64, + visit: &mut F, +) -> Result<(), MediaSourceError> { + match body { + EdgeBody::Once(bytes) => { + if to_u64(bytes.len()) != expected { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_body_length", + }); + } + visit(&bytes); + Ok(()) + } + EdgeBody::Stream(mut stream) => { + let mut seen = 0_u64; + while let Some(item) = stream.next().await { + let Ok(chunk) = item else { + return Err(MediaSourceError::Transport); + }; + let next = seen.saturating_add(to_u64(chunk.len())); + if next > expected { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_overlong_body", + }); + } + visit(&chunk); + seen = next; + } + if seen == expected { + Ok(()) + } else { + Err(MediaSourceError::Transport) + } + } + } +} + +/// SHA-256 of every byte of `source`'s pinned version, read once by one +/// pinned read in bounded chunks. A read that fails fails the digest; it is +/// never restarted or resumed. +async fn whole_digest(source: &VersionedMediaSource<'_>) -> Result<[u8; 32], MediaSourceError> { + let mut hasher = Sha256::new(); + let mut visit = |chunk: &[u8]| hasher.update(chunk); + source.stream_range(0, source.size(), &mut visit).await?; + let mut digest = [0_u8; 32]; + digest.copy_from_slice(&hasher.finalize()); + Ok(digest) +} + +/// `body` as sent to the reader, refused where it holds more or fewer than +/// `expected` bytes, so a reader never receives bytes of another length +/// under headers that describe this one. +fn exact_body(body: EdgeBody, expected: u64) -> Result { + match body { + EdgeBody::Once(bytes) if to_u64(bytes.len()) == expected => Ok(EdgeBody::Once(bytes)), + EdgeBody::Once(_) => Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_body_length", + }), + EdgeBody::Stream(stream) => Ok(EdgeBody::from_stream(futures::stream::unfold( + (stream, 0_u64, false), + move |(mut stream, sent, finished)| async move { + if finished { + return None; + } + match stream.next().await { + Some(Ok(chunk)) => { + let sent = sent.saturating_add(to_u64(chunk.len())); + if sent > expected { + let error = MediaSourceError::InvalidAnswer { + reason: "pinned_read_overlong_body", + }; + Some((Err(error), (stream, sent, true))) + } else { + Some((Ok(chunk), (stream, sent, false))) + } + } + Some(Err(_)) => Some((Err(MediaSourceError::Transport), (stream, sent, true))), + None if sent == expected => None, + None => Some((Err(MediaSourceError::Transport), (stream, sent, true))), + } + }, + ))), + } +} + +/// The first bytes of kept metadata for a signed version, naming its +/// layout. +const SIGNED_MAGIC: &[u8; 4] = b"TSL1"; + +/// The whole of the kept record for a version the lane declined: refused +/// by the signer, failed to read or hash, or out of time. +const DECLINED_MAGIC: &[u8; 4] = b"TSL0"; + +/// The longest metadata for a signed version is kept, whatever the origin +/// allows. +const SIGNED_MAX_LIFETIME: Duration = Duration::from_secs(3600); + +/// The longest a declined version is kept, so a source, policy or signer +/// that changes is tried again soon, while a burst of requests for one +/// version never hashes it once per request. +const DECLINED_MAX_LIFETIME: Duration = Duration::from_secs(60); + +/// How long a lookup may take before the origin's own answer, held unread +/// meanwhile, is no longer trusted to stream, and the bytes are read again +/// pinned. A lookup that waits on another caller's load can take as long as +/// that load. +const ORIGIN_BODY_MAX_WAIT: Duration = Duration::from_secs(1); + +/// The most signer headers kept metadata may hold. +const LARGE_MAX_HEADERS: usize = 64; + +/// What core keeps for a signed large version: never its bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +struct LargeRecord { + length: u64, + sha256: [u8; 32], + headers: Vec, +} + +/// What core keeps for one large version under one namespace. +#[derive(Debug, Clone, PartialEq, Eq)] +enum KeptOutcome { + /// The version was signed; its answers carry these headers. + Signed(LargeRecord), + /// The version was not signed; its answers go as the origin made them. + Declined, +} + +/// Metadata for a signed version as written: [`SIGNED_MAGIC`], the length, +/// the digest, the header count, then each header's name and value with +/// their lengths. +fn encode_large(record: &LargeRecord) -> Bytes { + let mut out = BytesMut::with_capacity(128); + out.extend_from_slice(SIGNED_MAGIC); + out.extend_from_slice(&record.length.to_be_bytes()); + out.extend_from_slice(&record.sha256); + let count = record.headers.len().min(LARGE_MAX_HEADERS); + out.extend_from_slice(&u32::try_from(count).unwrap_or(u32::MAX).to_be_bytes()); + for (name, value) in record.headers.iter().take(count) { + for field in [name.as_str().as_bytes(), value.as_bytes()] { + out.extend_from_slice(&u32::try_from(field.len()).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(field); + } + } + out.freeze() +} + +/// A kept outcome as [`encode_large`] or [`DECLINED_MAGIC`] wrote it, or +/// `None` for any other bytes. +fn decode_kept_outcome(bytes: &[u8]) -> Option { + if bytes == DECLINED_MAGIC.as_slice() { + return Some(KeptOutcome::Declined); + } + let mut reader = KeptReader { + rest: bytes.strip_prefix(SIGNED_MAGIC.as_slice())?, + }; + let length = reader.number()?; + let sha256: [u8; 32] = reader.take(32)?.try_into().ok()?; + let count = reader.length()?; + if count > LARGE_MAX_HEADERS { + return None; + } + let mut headers = Vec::with_capacity(count); + for _ in 0..count { + let name_length = reader.length()?; + let name = HeaderName::from_bytes(reader.take(name_length)?).ok()?; + let value_length = reader.length()?; + let value = HeaderValue::from_bytes(reader.take(value_length)?).ok()?; + if is_reserved_header(&name) { + return None; + } + headers.push((name, value)); + } + reader + .rest + .is_empty() + .then_some(KeptOutcome::Signed(LargeRecord { + length, + sha256, + headers, + })) +} + +/// Writes a signed version's headers on an answer of it: the signer's, +/// with a `Link` added to the origin's, and the whole source's +/// `Repr-Digest`. +fn decorate(headers: &mut HeaderMap, record: &LargeRecord) { + for (name, value) in &record.headers { + if name == header::LINK { + headers.append(name.clone(), value.clone()); + } else { + headers.insert(name.clone(), value.clone()); + } + } + let digest = format!( + "sha-256=:{}:", + base64::engine::general_purpose::STANDARD.encode(record.sha256) + ); + headers.insert( + HeaderName::from_static("repr-digest"), + HeaderValue::from_str(&digest).expect("should build a base64 digest"), + ); +} + +/// The answer where a signed or meant-to-be-signed version can no longer +/// be read: nothing of another version is sent in its place. +fn refusal() -> Response { + Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .header(header::CACHE_CONTROL, "no-store") + .header(header::RETRY_AFTER, "1") + .header(header::CONTENT_LENGTH, 0) + .body(EdgeBody::empty()) + .expect("should build a refusal") +} + +/// The headers a fresh origin answer for an unsigned version brings: its +/// status framing, type, coding, validators and redirect/authentication +/// semantics. Every other header stays as core prepared it for the reader. +const FRESH_REPRESENTATION_HEADERS: [&str; 16] = [ + "content-type", + "content-location", + "content-length", + "content-encoding", + "content-range", + "accept-ranges", + "etag", + "last-modified", + "repr-digest", + "content-digest", + "digest", + "content-md5", + "transfer-encoding", + "location", + "www-authenticate", + "retry-after", +]; + +/// Whether the origin says it serves byte ranges of this representation, +/// which every pinned read relies on. +fn advertises_byte_ranges(headers: &HeaderMap) -> bool { + headers + .get_all(header::ACCEPT_RANGES) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .any(|unit| unit.trim().eq_ignore_ascii_case("bytes")) +} + +/// The answer where the origin could not be asked again for an unsigned +/// version. +fn origin_unreachable() -> Response { + Response::builder() + .status(StatusCode::BAD_GATEWAY) + .header(header::CACHE_CONTROL, "no-store") + .header(header::CONTENT_LENGTH, 0) + .body(EdgeBody::empty()) + .expect("should build a gateway failure") +} + +/// An origin answer admitted to the large lane, with the version it pins. +pub(crate) struct LargeAnswer { + response: Response, + version: SourceVersion, + /// How long the origin let its answer be kept, shared, which bounds + /// how long its outcome is kept. + keep_for: Duration, +} + +impl LargeAnswer { + /// The origin's answer as it came, for a caller that cannot say how to + /// ask the origin again, sent unsigned. + pub(crate) fn into_origin_response(self) -> Response { + self.response + } +} + +/// The one wall-clock budget for a large source's namespace resolution, +/// cache waits, hashing and signing. +#[derive(Debug, Clone, Copy)] +struct LargeBudget { + started: Instant, + total: Duration, +} + +impl LargeBudget { + /// What is left of the budget now. + fn remaining(self) -> Duration { + self.total.saturating_sub(self.started.elapsed()) + } +} + +/// The public addresses a large source may be named by, worked out before +/// anything is hashed. +#[derive(Debug, Clone, PartialEq, Eq)] +struct LargeReferences { + /// The fetched address, where it is `https`, carries no user + /// information, query or fragment, and core does not sign the request + /// for it: an address core signs is a private one and is never named. + source: Option, + /// The reader's address, where it is `https` and the reader's request + /// carried no query. + reader: Option, +} + +impl LargeReferences { + fn of(identity: &VariantIdentity<'_>, origin_signed: bool) -> Self { + let source = if origin_signed { + None + } else { + source_reference(identity.fetched_url) + .filter(|reference| reference.starts_with("https://")) + .map(str::to_owned) + }; + let reader = (identity.reader_query_free && identity.reader_url.starts_with("https://")) + .then(|| identity.reader_url.to_owned()); + Self { source, reader } + } + + fn any(&self) -> bool { + self.source.is_some() || self.reader.is_some() + } +} + +/// Everything the hashing and signing of one large version owns, because +/// the loading cache keeps the load with its key rather than borrowing the +/// request, and runs it once for every caller asking for that key at the +/// same time. +struct LargeLoad { + signer: Arc, + /// The request's services without its connection, since the outcome + /// serves every caller waiting on the load. + services: RuntimeServices, + url: url::Url, + headers: HeaderMap, + backend_name: String, + auth: Option, + unsigned_headers: Vec, + version: SourceVersion, + reader_url: String, + content_type: String, + references: LargeReferences, + /// The namespace the outcome is kept under, which the signer must sign + /// under. + namespace: String, + budget: LargeBudget, + keep_for: Duration, +} + +impl LargeLoad { + fn into_loader(self) -> Loader { + Box::new(move || Box::pin(async move { Ok(self.run().await) })) + } + + /// Hashes and signs within the budget, giving the outcome to keep. A + /// declined outcome is kept briefly too, so the version is not hashed + /// again for every request while it stands. + async fn run(self) -> LoadedValue { + let signer_id = self.signer.signer_id(); + let remaining = self.budget.remaining(); + let outcome = match self.services.timer() { + Some(_) if remaining.is_zero() => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + None + } + Some(timer) => { + match futures::future::select(Box::pin(self.sign()), timer.sleep(remaining)).await { + Either::Left((outcome, _)) => outcome, + Either::Right(((), _)) => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + None + } + } + } + None => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=no_timer" + ); + None + } + }; + match outcome { + Some(record) => LoadedValue::new(encode_large(&record)) + .with_lifetime(self.keep_for.min(SIGNED_MAX_LIFETIME)), + None => LoadedValue::new(Bytes::from_static(DECLINED_MAGIC)) + .with_lifetime(self.keep_for.min(DECLINED_MAX_LIFETIME)), + } + } + + /// Hashes the pinned version once and asks the signer for it. + async fn sign(&self) -> Option { + let signer_id = self.signer.signer_id(); + let declined = |reason: &str| { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason={reason}" + ); + }; + let origin = PinnedOrigin { + url: &self.url, + headers: &self.headers, + backend_name: &self.backend_name, + auth: self.auth.as_ref(), + unsigned_headers: &self.unsigned_headers, + response_header_rewrite: None, + }; + let source = VersionedMediaSource::pinned(&self.services, origin, self.version.clone()); + let digest = whole_digest(&source) + .await + .map_err(|error| declined(error.code())) + .ok()?; + let Some(origin_host) = self.url.host_str().map(str::to_ascii_lowercase) else { + declined("no_origin_host"); + return None; + }; + let remaining = self.budget.remaining(); + if remaining.is_zero() { + declined("timeout"); + return None; + } + let source_sha256 = hex::encode(digest); + let job = LargeMediaJob { + reader_url: &self.reader_url, + content_type: &self.content_type, + source: &source, + source_sha256: &source_sha256, + source_reference: self.references.source.as_deref(), + reader_reference: self.references.reader.as_deref(), + source_origin_host: &origin_host, + remaining, + }; + let signed = match self.signer.sign_large(&self.services, &job).await { + Ok(signed) + if signed + .headers + .iter() + .any(|(name, _)| is_reserved_header(name)) => + { + declined("reserved_header"); + return None; + } + Ok(signed) if signed.headers.len() > LARGE_MAX_HEADERS => { + declined("too_many_headers"); + return None; + } + Ok(signed) if signed.namespace != self.namespace => { + declined("namespace_changed"); + return None; + } + Ok(signed) => signed, + Err(failure) => { + declined(failure.reason); + return None; + } + }; + log::info!( + "media signing: outcome=signed lane=large signer={signer_id} source_bytes={}", + source.size() + ); + Some(LargeRecord { + length: source.size(), + sha256: digest, + headers: signed.headers, + }) + } +} + +impl MediaClaim { + /// Whether this claim may use the large lane at all: the signer signs + /// large sources and the request is shared. + fn large_lane_open(&self) -> bool { + self.shared && self.signer.max_large_source_bytes() > 0 + } + + /// The version a signable origin answer of `total` bytes, larger than + /// the bounded limit, pins in the large lane, or why it cannot. The + /// answer's coding, privacy, transform rule, type and any manifest + /// `Link` of the origin's own were already checked, see + /// `unsignable_reason`. Every check here is cheap and runs before + /// anything is hashed. + fn large_version( + &self, + services: &RuntimeServices, + headers: &HeaderMap, + total: Option, + facts: &OriginFacts, + ) -> Result { + let ceiling = self.signer.max_large_source_bytes(); + if ceiling == 0 { + return Err("exceeds_limit"); + } + if !self.shared { + return Err("large_request_not_shared"); + } + // An outcome that cannot be kept would be worked out again for + // every request, each range of one playback included. + if facts.keep_for.is_none() || !services.loading_cache_for(ASSETS_CACHE).keeps_values() { + return Err("large_outcome_not_kept"); + } + let Some(length) = total else { + return Err("unknown_length"); + }; + if length > ceiling { + return Err("exceeds_large_limit"); + } + if !services.http_client().supports_streaming_responses() { + return Err("no_streaming_transport"); + } + if self.signer.max_large_media_duration().is_zero() { + return Err("no_large_media_budget"); + } + if services.timer().is_none() { + return Err("no_timer"); + } + SourceVersion::of(headers, length).ok_or("no_strong_etag") + } + + /// What a signable origin answer of `total` bytes, larger than the + /// bounded limit, becomes: a [`Delivery::Large`] where the large lane + /// admits it, else the origin's answer as it came. + pub(super) fn large_or_unsigned( + &self, + services: &RuntimeServices, + response: Response, + total: Option, + facts: &OriginFacts, + ) -> Delivery { + match self.large_version(services, response.headers(), total, facts) { + Ok(version) => Delivery::Large(LargeAnswer { + response, + version, + keep_for: facts.keep_for.unwrap_or_default(), + }), + Err(reason) => { + log::info!( + "media signing: outcome=unsigned signer={} reason={reason}", + self.signer.signer_id() + ); + Delivery::Send(response) + } + } + } + + /// Answers the reader from an origin answer admitted to the large lane. + /// + /// A source no public address can name, or whose signer's namespace + /// cannot be resolved, goes as the origin answered it, before anything + /// is hashed. An outcome kept for the pinned version under the signer's + /// current namespace answers next: a signed one decorates the origin's + /// own answer, a declined one leaves it as it came. A `HEAD`, or an + /// answer whose origin does not say it serves byte ranges, never starts + /// a load and goes as it came. Otherwise the version is hashed and + /// signed once, through the loading cache, so callers asking at the + /// same time share that one load. A signed version is answered by + /// pinned reads, refused where the origin no longer serves it; a + /// declined one by one fresh origin answer to the request as core first + /// sent it, since no signature binds its bytes. `origin` is how the + /// origin is asked again. + pub(crate) async fn deliver_large( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + origin: PinnedOrigin<'_>, + answer: LargeAnswer, + ) -> Response { + let signer_id = self.signer.signer_id(); + let budget = LargeBudget { + started: Instant::now(), + total: self.signer.max_large_media_duration(), + }; + let LargeAnswer { + response, + version, + keep_for, + } = answer; + let references = LargeReferences::of(identity, origin.auth.is_some()); + if !references.any() { + log::info!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=no_public_reference" + ); + return response; + } + let Some(timer) = services.timer() else { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=no_timer" + ); + return response; + }; + let remaining = budget.remaining(); + if remaining.is_zero() { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + return response; + } + let namespace = match futures::future::select( + self.signer.namespace(services), + timer.sleep(remaining), + ) + .await + { + Either::Left((Ok(namespace), _)) => namespace, + Either::Left((Err(failure), _)) => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason={failure}" + ); + return response; + } + Either::Right(((), _)) => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + return response; + } + }; + let key = self.large_store_key(services, &namespace, identity, version.etag()); + let looked_up = Instant::now(); + let kept = match futures::future::select( + Box::pin(self.large_kept(services, &key, Some(version.length()))), + timer.sleep(budget.remaining()), + ) + .await + { + Either::Left((kept, _)) => kept, + Either::Right(((), _)) => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + if looked_up.elapsed() <= ORIGIN_BODY_MAX_WAIT { + return response; + } + let (parts, _) = response.into_parts(); + return self.deliver_fresh(services, origin, parts).await; + } + }; + if let Some(kept) = kept { + if looked_up.elapsed() <= ORIGIN_BODY_MAX_WAIT { + log::debug!("media signing: lane=large signer={signer_id} from=kept_outcome"); + return match kept { + KeptOutcome::Signed(record) => { + let (mut parts, body) = response.into_parts(); + decorate(&mut parts.headers, &record); + Response::from_parts(parts, body) + } + KeptOutcome::Declined => response, + }; + } + // The origin's answer waited unread for the lookup, so it is no + // longer trusted to stream. + let (parts, _) = response.into_parts(); + return match kept { + KeptOutcome::Signed(record) => { + let source = VersionedMediaSource::pinned(services, origin, version); + self.deliver_pinned(&source, parts, &record).await + } + KeptOutcome::Declined => self.deliver_fresh(services, origin, parts).await, + }; + } + // Only a reader's own `GET` of a version the origin serves in byte + // ranges starts the work of hashing it. + if self.reader_method == Method::HEAD { + log::debug!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=head_starts_no_load" + ); + return response; + } + if !advertises_byte_ranges(response.headers()) { + log::info!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=origin_ranges_not_advertised" + ); + return response; + } + if budget.remaining().is_zero() { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + if looked_up.elapsed() <= ORIGIN_BODY_MAX_WAIT { + return response; + } + let (parts, _) = response.into_parts(); + return self.deliver_fresh(services, origin, parts).await; + } + // The origin's own body is not read: the load reads the version + // pinned, so it serves every caller waiting on it. + let (parts, _) = response.into_parts(); + let load = LargeLoad { + signer: Arc::clone(&self.signer), + services: services.clone().without_connection(), + url: origin.url.clone(), + headers: origin.headers.clone(), + backend_name: origin.backend_name.to_owned(), + auth: origin.auth.cloned(), + unsigned_headers: origin.unsigned_headers.to_vec(), + version: version.clone(), + reader_url: identity.reader_url.to_owned(), + content_type: header_text(&parts.headers, &header::CONTENT_TYPE).to_owned(), + references, + namespace, + budget, + keep_for, + }; + let loaded = match futures::future::select( + Box::pin(services.load_cached(&self.spec, &key, load.into_loader())), + timer.sleep(budget.remaining()), + ) + .await + { + Either::Left((loaded, _)) => loaded.ok().and_then(|bytes| decode_kept_outcome(&bytes)), + Either::Right(((), _)) => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + None + } + }; + match loaded { + Some(KeptOutcome::Signed(record)) if record.length == version.length() => { + let source = VersionedMediaSource::pinned(services, origin, version); + self.deliver_pinned(&source, parts, &record).await + } + _ => self.deliver_fresh(services, origin, parts).await, + } + } + + /// Answers the reader of a version the lane does not sign with one + /// fresh origin answer to the request as core first sent it, the + /// reader's own range and conditions included, its origin signing + /// renewed. Nothing is retried. + /// + /// The status, body and every header describing the representation + /// are the fresh answer's, so a changed version goes with its own + /// validators. The rest are `parts`, the headers core and its caller + /// prepared for this reader, to which the fresh answer's `Link` and + /// `Content-Security-Policy` values are added, never replacing the + /// caller's: provenance the origin now declares is kept, and several + /// policies only narrow what a reader allows. The caller's cache policy + /// describes the first answer, not this one, so the answer is held by + /// no cache at all. A cookie or any other header the fresh answer + /// sets is never passed on, since the caller's own handling never saw + /// it. + async fn deliver_fresh( + &self, + services: &RuntimeServices, + origin: PinnedOrigin<'_>, + mut parts: http::response::Parts, + ) -> Response { + let signer_id = self.signer.signer_id(); + let Ok(mut request) = origin.signed_request(&self.reader_method, origin.base_headers()) + else { + log::warn!( + "media signing: outcome=unanswered lane=large signer={signer_id} reason=origin_request_unsigned" + ); + return origin_unreachable(); + }; + if services.http_client().supports_streaming_responses() { + request = request.with_stream_response(); + } + // The platform's error may name the address, which can carry a + // credential, so it is not logged. + let Ok(fresh) = services.http_client().send(request).await else { + log::warn!( + "media signing: outcome=unanswered lane=large signer={signer_id} reason=origin_unreachable" + ); + return origin_unreachable(); + }; + let (mut fresh, body) = fresh.response.into_parts(); + // Fresh cookies are not delivered. The existing redirect-loop guard + // must not assume they will change the reader's next request. + fresh.headers.remove(header::SET_COOKIE); + if let Some(rewrite) = origin.response_header_rewrite { + crate::response_header_rewrite::rewrite_origin_urls_in_headers( + &mut fresh.headers, + &rewrite, + ); + } + parts.status = fresh.status; + for name in FRESH_REPRESENTATION_HEADERS { + parts.headers.remove(name); + for value in fresh.headers.get_all(name) { + parts.headers.append(name, value.clone()); + } + } + for name in [header::LINK, header::CONTENT_SECURITY_POLICY] { + for value in fresh.headers.get_all(&name) { + if !parts + .headers + .get_all(&name) + .iter() + .any(|kept| kept == value) + { + parts.headers.append(name.clone(), value.clone()); + } + } + } + let mut response = Response::from_parts(parts, body); + crate::cache_policy::remove_owned_cache_headers(&mut response); + let headers = response.headers_mut(); + crate::cache_policy::apply_no_store_private_to_headers(headers); + for name in [header::EXPIRES, header::AGE] { + headers.remove(name); + } + response + } + + /// Decorates the origin's `304` for a large version kept as signed + /// under the signer's current namespace, so a reader revalidating it is + /// told of its manifest. Any other `304` goes as it came. + pub(super) async fn decorate_not_modified( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + facts: &OriginFacts, + response: Response, + ) -> Response { + // A `304` whose own `Link` already names a manifest keeps the + // origin's provenance as it came; no kept outcome is added to it. + if !self.large_lane_open() + || !self.looks_like_media + || facts.private + || carries_manifest_link(response.headers()) + { + return response; + } + let Some(version) = SourceVersion::of(response.headers(), 0) else { + return response; + }; + let Some(timer) = services.timer() else { + return response; + }; + let budget = LargeBudget { + started: Instant::now(), + total: self.signer.max_large_media_duration(), + }; + let remaining = budget.remaining(); + if remaining.is_zero() { + return response; + } + let namespace = + match futures::future::select(self.signer.namespace(services), timer.sleep(remaining)) + .await + { + Either::Left((Ok(namespace), _)) => namespace, + _ => return response, + }; + let key = self.large_store_key(services, &namespace, identity, version.etag()); + let remaining = budget.remaining(); + if remaining.is_zero() { + return response; + } + let record = match futures::future::select( + Box::pin(self.large_kept(services, &key, None)), + timer.sleep(remaining), + ) + .await + { + Either::Left((Some(KeptOutcome::Signed(record)), _)) => record, + _ => return response, + }; + let (mut parts, body) = response.into_parts(); + decorate(&mut parts.headers, &record); + Response::from_parts(parts, body) + } + + /// Answers the reader's preconditions and range against the pinned, + /// signed version, reading the bytes it needs by pinned reads, on + /// `parts`, the origin's headers as core prepared them, and adds the + /// signed version's headers from `record`. A version the origin no + /// longer serves is refused, never replaced, since the headers bind + /// these exact bytes. + async fn deliver_pinned( + &self, + source: &VersionedMediaSource<'_>, + mut parts: http::response::Parts, + record: &LargeRecord, + ) -> Response { + let length = source.size(); + let outcome = evaluate( + &self.reader_method, + &self.preconditions, + Some(source.version().etag()), + None, + length, + ); + let carries_body = self.reader_method != Method::HEAD; + let range = match outcome { + Outcome::Full if carries_body => length.checked_sub(1).map(|last| (0, last)), + Outcome::Partial { first, last } => Some((first, last)), + _ => None, + }; + let body = match range { + Some((first, last)) => { + let read = source.send_pinned(first, last).await; + match read.and_then(|body| exact_body(body, last - first + 1)) { + Ok(body) => body, + Err(error) => { + log::warn!( + "media signing: outcome=refused lane=large signer={} reason={}", + self.signer.signer_id(), + error.code() + ); + return refusal(); + } + } + } + None => EdgeBody::empty(), + }; + let headers = &mut parts.headers; + for name in [ + "content-range", + "content-length", + "transfer-encoding", + "content-encoding", + ] { + headers.remove(name); + } + match outcome { + Outcome::Full => { + parts.status = StatusCode::OK; + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(length)); + } + Outcome::Partial { first, last } => { + parts.status = StatusCode::PARTIAL_CONTENT; + headers.insert( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes {first}-{last}/{length}")) + .expect("should build a content range"), + ); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(last - first + 1)); + } + Outcome::NotModified => { + parts.status = StatusCode::NOT_MODIFIED; + } + Outcome::PreconditionFailed => { + parts.status = StatusCode::PRECONDITION_FAILED; + headers.remove(header::CONTENT_TYPE); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); + } + Outcome::Unsatisfiable => { + parts.status = StatusCode::RANGE_NOT_SATISFIABLE; + headers.insert( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes */{length}")) + .expect("should build a content range"), + ); + headers.remove(header::CONTENT_TYPE); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); + } + } + if matches!( + outcome, + Outcome::Full | Outcome::Partial { .. } | Outcome::NotModified + ) { + decorate(headers, record); + } + Response::from_parts(parts, body) + } + + /// The outcome kept under `key`, or `None`. A signed outcome whose + /// length is not `length`, where an answer states one, or a record that + /// does not read back, is dropped. The lookup itself runs no load, but + /// waits on one another caller is running for `key`. + async fn large_kept( + &self, + services: &RuntimeServices, + key: &StoreKey, + length: Option, + ) -> Option { + if !services.loading_cache_for(ASSETS_CACHE).keeps_values() { + return None; + } + let bytes = services + .load_cached( + &self.spec, + key, + Box::new(|| Box::pin(async { Err(LoadError::new(super::NOT_KEPT)) })), + ) + .await + .ok()?; + let kept = decode_kept_outcome(&bytes).filter(|kept| match kept { + KeptOutcome::Signed(record) => length.is_none_or(|length| record.length == length), + KeptOutcome::Declined => true, + }); + if kept.is_none() { + log::warn!( + "media signing: lane=large kept outcome unreadable or for another length signer={}", + self.signer.signer_id() + ); + services + .loading_cache_for(ASSETS_CACHE) + .remove(&services.clone().without_connection(), &self.spec, key) + .await; + } + kept + } + + /// The key the outcome for `identity`'s version `etag` is kept under, + /// bound to the publisher's prefix, the signer and its `namespace`. + fn large_store_key( + &self, + services: &RuntimeServices, + namespace: &str, + identity: &VariantIdentity<'_>, + etag: &str, + ) -> StoreKey { + let mut digest = Sha256::new(); + for part in [ + self.signer.signer_id(), + namespace, + identity.reader_url, + identity.origin_identity, + identity.fetched_url, + etag, + ] { + digest.update(part.as_bytes()); + digest.update([0]); + } + StoreKey::new( + services.server_key_prefix(), + ASSETS_CACHE, + format!("media-large-{}", hex::encode(digest.finalize().as_slice())), + ) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/media_signing/large/tests.rs b/crates/trusted-server-core/src/media_signing/large/tests.rs new file mode 100644 index 0000000000..d6e72274f7 --- /dev/null +++ b/crates/trusted-server-core/src/media_signing/large/tests.rs @@ -0,0 +1,2020 @@ +use std::sync::{Arc, Mutex}; +use std::time::SystemTime; + +use base64::Engine as _; +use error_stack::Report; +use futures::FutureExt as _; +use futures::StreamExt as _; +use futures::channel::oneshot; +use futures::executor::block_on; +use sha2::Digest as _; + +use super::*; +use crate::error::TrustedServerError; +use crate::media_signing::test_support::MANIFEST_LINK; +use crate::media_signing::{Answered, MediaJob, MediaNamespacing, MediaSigning}; +use crate::origin_auth::OriginSigner; +use crate::platform::test_support::{ + ElapsedTimer, MemoryLoadingCache, StoppedTimer, build_services_with_http_client, +}; +use crate::platform::{ + CacheSpec, PlatformError, PlatformHttpClient, PlatformLoadingCache, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, PlatformTimer, +}; +use crate::test_support::tests::create_test_settings; + +const PATH: &str = "/media/42"; +const FETCHED_URL: &str = "https://origin.example.com/media/42"; +const QUERY_FETCHED_URL: &str = "https://origin.example.com/media/42?v=2"; +const READER_URL: &str = "https://www.example.com/media/42"; +const ORIGIN_IDENTITY: &str = "https://origin.example.com\0origin.example.com"; +const ETAG_V1: &str = "\"v1\""; +const ETAG_V2: &str = "\"v2\""; +/// The size of each chunk the stub origin streams, so every body arrives +/// in several pieces. +const CHUNK: usize = 5; +/// The `Authorization` the test origin signer writes. +const ORIGIN_SIGNATURE: &str = "TestOrigin signed"; + +fn source() -> Vec { + (0_u8..64).collect() +} + +fn other_source() -> Vec { + (100_u8..164).collect() +} + +fn identity() -> VariantIdentity<'static> { + VariantIdentity { + reader_url: READER_URL, + origin_identity: ORIGIN_IDENTITY, + fetched_url: FETCHED_URL, + reader_query_free: true, + } +} + +/// An origin answer for `bytes` at version `etag`: the bytes `range` +/// names as a `206`, or all of them as a `200`, streamed in [`CHUNK`]s and +/// cut with a stream error after `cut_at` bytes where given. +fn serve( + bytes: &[u8], + etag: &str, + range: Option<(u64, u64)>, + cut_at: Option, +) -> Response { + let total = bytes.len(); + let (status, slice) = match range { + Some((first, last)) => { + let first = usize::try_from(first).expect("should fit a test offset"); + let last = usize::try_from(last).expect("should fit a test offset"); + (StatusCode::PARTIAL_CONTENT, bytes[first..=last].to_vec()) + } + None => (StatusCode::OK, bytes.to_vec()), + }; + let mut builder = Response::builder() + .status(status) + .header(header::CONTENT_TYPE, "image/png") + .header(header::ETAG, etag) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header(header::ACCEPT_RANGES, "bytes") + .header(header::CONTENT_LENGTH, slice.len()); + if let Some((first, last)) = range { + builder = builder.header( + header::CONTENT_RANGE, + format!("bytes {first}-{last}/{total}"), + ); + } + let kept = cut_at.map_or(slice.len(), |cut| cut.min(slice.len())); + let mut items: Vec> = slice[..kept] + .chunks(CHUNK) + .map(|chunk| Ok(Bytes::copy_from_slice(chunk))) + .collect(); + if cut_at.is_some() { + items.push(Err(std::io::Error::other("origin connection cut"))); + } + builder + .body(EdgeBody::from_stream(futures::stream::iter(items))) + .expect("should build an origin answer") +} + +/// The origin's whole `200` for the reader's first request. +fn whole() -> Response { + serve(&source(), ETAG_V1, None, None) +} + +/// The origin's answer to a `HEAD`: the whole's headers, no body. +fn head_answer(bytes: &[u8], etag: &str) -> Response { + let (parts, _) = serve(bytes, etag, None, None).into_parts(); + Response::from_parts(parts, EdgeBody::empty()) +} + +fn not_modified(etag: &str) -> Response { + Response::builder() + .status(StatusCode::NOT_MODIFIED) + .header(header::ETAG, etag) + .header(header::CACHE_CONTROL, "public, max-age=600") + .body(EdgeBody::empty()) + .expect("should build a not modified answer") +} + +fn assert_unchanged_not_modified(delivery: Delivery, headers: &HeaderMap) { + let Delivery::Send(response) = delivery else { + panic!("should send the original not modified response"); + }; + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + assert_eq!(response.headers(), headers, "should preserve every header"); + let (_, body) = collect(response); + assert_eq!(body, Some(Vec::new()), "should preserve the empty body"); +} + +/// A `bytes=first-last` request range. +fn requested_range(headers: &HeaderMap) -> Option<(u64, u64)> { + let value = headers.get(header::RANGE)?.to_str().ok()?; + let (first, last) = value.strip_prefix("bytes=")?.split_once('-')?; + Some((first.parse().ok()?, last.parse().ok()?)) +} + +struct VersionChange { + after: usize, + bytes: Vec, + etag: String, + headers: Vec<(HeaderName, HeaderValue)>, +} + +#[derive(Default)] +struct OriginState { + bytes: Vec, + etag: String, + /// From the request after this many on, the origin serves the other + /// version, answering it with these headers too. + flip: Option, + /// Headers every answer carries besides [`serve`]'s own, replacing + /// them. + answer_headers: Vec<(HeaderName, HeaderValue)>, + /// Override the status after this many requests. + status_after: Option<(usize, StatusCode)>, + /// The next answer's stream is cut after this many bytes. + cut_next: Option, + sends: usize, + wrong_content_range: bool, + requests: Vec, +} + +/// An origin that honors `Range` and `If-Match` and can change version. +struct RangeOrigin { + state: Mutex, +} + +impl RangeOrigin { + fn new(bytes: &[u8], etag: &str) -> Arc { + Arc::new(Self { + state: Mutex::new(OriginState { + bytes: bytes.to_vec(), + etag: etag.to_owned(), + ..OriginState::default() + }), + }) + } + + /// Serves `bytes` at `etag` from the request after the `after`th on. + fn flip_after(&self, after: usize, bytes: &[u8], etag: &str) { + self.flip_after_with(after, bytes, etag, Vec::new()); + } + + /// As [`Self::flip_after`], the new version's answers carrying + /// `headers` too. + fn flip_after_with( + &self, + after: usize, + bytes: &[u8], + etag: &str, + headers: Vec<(HeaderName, HeaderValue)>, + ) { + self.state.lock().expect("should lock origin").flip = Some(VersionChange { + after, + bytes: bytes.to_vec(), + etag: etag.to_owned(), + headers, + }); + } + + fn cut_next_at(&self, at: usize) { + self.state.lock().expect("should lock origin").cut_next = Some(at); + } + + fn answer_wrong_content_range(&self) { + self.state + .lock() + .expect("should lock origin") + .wrong_content_range = true; + } + + fn requests(&self) -> Vec { + self.state + .lock() + .expect("should lock origin") + .requests + .clone() + } + + /// The ranges every request asked for, in order. + fn ranges(&self) -> Vec> { + self.requests().iter().map(requested_range).collect() + } + + /// How many requests read the whole version, which only hashing does + /// besides a whole delivery. + fn whole_reads(&self) -> usize { + self.ranges() + .iter() + .filter(|range| **range == Some((0, 63))) + .count() + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformHttpClient for RangeOrigin { + fn supports_streaming_responses(&self) -> bool { + true + } + + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + let mut state = self.state.lock().expect("should lock origin"); + if state + .flip + .as_ref() + .is_some_and(|flip| state.sends >= flip.after) + { + let flip = state.flip.take().expect("should have a due version change"); + state.bytes = flip.bytes; + state.etag = flip.etag; + state.answer_headers = flip.headers; + } + state.sends += 1; + let headers = request.request.headers().clone(); + state.requests.push(headers.clone()); + if let Some(if_match) = headers.get(header::IF_MATCH) + && if_match.as_bytes() != state.etag.as_bytes() + { + let refused = Response::builder() + .status(StatusCode::PRECONDITION_FAILED) + .header(header::ETAG, state.etag.as_str()) + .body(EdgeBody::empty()) + .expect("should build a refusal"); + return Ok(PlatformResponse::new(refused)); + } + let range = requested_range(&headers); + let cut = state.cut_next.take(); + let mut response = serve(&state.bytes, &state.etag, range, cut); + if let Some((after, status)) = state.status_after + && state.sends > after + { + *response.status_mut() = status; + } + for (name, value) in &state.answer_headers { + response.headers_mut().insert(name.clone(), value.clone()); + } + if state.wrong_content_range && range.is_some() { + response.headers_mut().insert( + header::CONTENT_RANGE, + HeaderValue::from_static("bytes 0-0/1"), + ); + } + Ok(PlatformResponse::new(response)) + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } +} + +/// An origin signer that writes [`ORIGIN_SIGNATURE`] and owns every +/// `x-test-origin-*` header. +struct TestOriginSigner; + +impl OriginSigner for TestOriginSigner { + fn sign( + &self, + _method: &Method, + _url: &url::Url, + headers: &mut HeaderMap, + _now: SystemTime, + ) -> Result<(), Report> { + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static(ORIGIN_SIGNATURE), + ); + Ok(()) + } + + fn remove_namespace_headers(&self, headers: &mut HeaderMap) { + headers.remove("x-test-origin-key"); + } +} + +/// One large job the stub signer was handed. +#[derive(Debug, Clone)] +struct LargeJobRecord { + source_sha256: String, + size: u64, + remaining: Duration, + source_reference: Option, + reader_reference: Option, + origin_host: String, +} + +/// How the stub signer answers a large job. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LargeBehavior { + /// Signs, adding [`MANIFEST_LINK`]. + Sign, + /// Signs after confirming the pinned version is still current. + ConfirmThenSign, + /// Refuses, as a source with provenance of its own is refused. + Refuse, + /// Never answers. + Hang, +} + +/// A signer whose bounded limit is 16 bytes and whose large lane takes up +/// to 1 KiB. +struct LargeSigner { + behavior: LargeBehavior, + namespace: Mutex>, + namespace_gate: Mutex>>, + large_duration: Duration, + attempts: Mutex, + jobs: Mutex>, + failures: Mutex>, +} + +impl LargeSigner { + fn new(behavior: LargeBehavior) -> Arc { + Arc::new(Self { + behavior, + namespace: Mutex::new(Ok("policy-1".to_owned())), + namespace_gate: Mutex::new(None), + large_duration: Duration::from_secs(60), + attempts: Mutex::new(0), + jobs: Mutex::new(Vec::new()), + failures: Mutex::new(Vec::new()), + }) + } + + fn set_namespace(&self, namespace: Result) { + *self.namespace.lock().expect("should lock namespace") = namespace; + } + + /// How many times core called `sign_large`. + fn attempts(&self) -> usize { + *self.attempts.lock().expect("should lock attempts") + } + + fn jobs(&self) -> Vec { + self.jobs.lock().expect("should lock jobs").clone() + } + + fn failures(&self) -> Vec<&'static str> { + self.failures.lock().expect("should lock failures").clone() + } +} + +impl MediaSigner for LargeSigner { + fn signer_id(&self) -> &'static str { + "testing.large-media" + } + + fn handles_path(&self, path: &str) -> bool { + !path.starts_with("/api/") + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type.starts_with("image/") + } + + fn max_source_bytes(&self) -> usize { + 16 + } + + fn namespace<'a>(&'a self, _services: &'a RuntimeServices) -> MediaNamespacing<'a> { + let namespace = self + .namespace + .lock() + .expect("should lock namespace") + .clone(); + let gate = self + .namespace_gate + .lock() + .expect("should lock namespace gate") + .take(); + Box::pin(async move { + if let Some(gate) = gate { + gate.await.expect("should release namespace"); + } + namespace + }) + } + + fn sign<'a>( + &'a self, + _services: &'a RuntimeServices, + _job: &'a MediaJob<'a>, + ) -> MediaSigning<'a> { + Box::pin(async { + Err(MediaSigningFailure { + reason: "bounded_not_expected", + }) + }) + } + + fn max_large_source_bytes(&self) -> u64 { + 1024 + } + + fn max_large_media_duration(&self) -> Duration { + self.large_duration + } + + fn sign_large<'a>( + &'a self, + _services: &'a RuntimeServices, + job: &'a LargeMediaJob<'a>, + ) -> LargeMediaSigning<'a> { + Box::pin(async move { + *self.attempts.lock().expect("should lock attempts") += 1; + match self.behavior { + LargeBehavior::Refuse => { + return Err(MediaSigningFailure { + reason: "source_has_embedded_provenance", + }); + } + LargeBehavior::Hang => futures::future::pending::<()>().await, + LargeBehavior::ConfirmThenSign => { + if let Err(error) = job.source.confirm_current().await { + self.failures + .lock() + .expect("should lock failures") + .push(error.code()); + return Err(MediaSigningFailure { + reason: error.code(), + }); + } + } + LargeBehavior::Sign => {} + } + self.jobs + .lock() + .expect("should lock jobs") + .push(LargeJobRecord { + source_sha256: job.source_sha256.to_owned(), + size: job.source.size(), + remaining: job.remaining, + source_reference: job.source_reference.map(str::to_owned), + reader_reference: job.reader_reference.map(str::to_owned), + origin_host: job.source_origin_host.to_owned(), + }); + let namespace = self + .namespace + .lock() + .expect("should lock namespace") + .clone() + .unwrap_or_default(); + Ok(LargeSignedMedia { + headers: vec![(header::LINK, HeaderValue::from_static(MANIFEST_LINK))], + namespace, + }) + }) + } +} + +#[derive(Default)] +struct PendingLargeCache { + calls: Mutex, +} + +#[async_trait::async_trait(?Send)] +impl PlatformLoadingCache for PendingLargeCache { + async fn get_or_load( + &self, + _services: &RuntimeServices, + _spec: &CacheSpec, + _key: &StoreKey, + _load: Loader, + ) -> Result { + *self.calls.lock().expect("should lock cache calls") += 1; + futures::future::pending().await + } + + async fn remove(&self, _services: &RuntimeServices, _spec: &CacheSpec, _key: &StoreKey) { + panic!("should never remove a pending cache outcome"); + } +} + +fn services_with_timer( + signer: &Arc, + origin: &Arc, + timer: Arc, +) -> RuntimeServices { + build_services_with_http_client(Arc::clone(origin) as Arc) + .with_media_signer(Some(Arc::clone(signer) as Arc)) + .with_loading_cache(Arc::new(MemoryLoadingCache::default())) + .with_timer(timer) +} + +fn services_for(signer: &Arc, origin: &Arc) -> RuntimeServices { + services_with_timer(signer, origin, Arc::new(StoppedTimer)) +} + +fn claim_for( + services: &RuntimeServices, + method: &Method, + pairs: &[(HeaderName, &str)], +) -> MediaClaim { + let mut headers = HeaderMap::new(); + headers.insert("sec-fetch-dest", HeaderValue::from_static("image")); + for (name, value) in pairs { + headers.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build a header"), + ); + } + MediaClaim::of_request( + &create_test_settings(), + services, + method, + PATH, + &headers, + true, + ) + .expect("should claim a media path") +} + +/// How core asks the origin again in a test: the address, the request's +/// headers before signing, and the origin's signing where it has one. +struct TestOrigin<'a> { + identity: VariantIdentity<'a>, + headers: HeaderMap, + auth: Option, + unsigned_headers: Vec, + response_header_rewrite: Option>, +} + +impl TestOrigin<'static> { + /// An origin request that forwarded the reader's `pairs`, as a route + /// forwards a reader's range and conditions. + fn forwarding(pairs: &[(HeaderName, &str)]) -> Self { + let mut headers = HeaderMap::new(); + for (name, value) in pairs { + headers.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build a header"), + ); + } + Self { + headers, + ..Self::plain() + } + } + + fn plain() -> Self { + Self { + identity: identity(), + headers: HeaderMap::new(), + auth: None, + unsigned_headers: Vec::new(), + response_header_rewrite: None, + } + } +} + +/// Delivers the origin's `response` through `origin`, by the large lane +/// where core admits it. Whether it was admitted, and the answer. +fn run_through( + claim: &MediaClaim, + services: &RuntimeServices, + origin: &TestOrigin<'_>, + response: Response, +) -> (bool, Response) { + block_on(async { + let facts = OriginFacts::of(response.headers()); + match claim + .deliver( + services, + &origin.identity, + Answered::Origin(facts), + response, + ) + .await + { + Delivery::Large(answer) => { + let url = url::Url::parse(origin.identity.fetched_url) + .expect("should parse the fetched address"); + let pinned = PinnedOrigin { + url: &url, + headers: &origin.headers, + backend_name: "origin", + auth: origin.auth.as_ref(), + unsigned_headers: &origin.unsigned_headers, + response_header_rewrite: origin.response_header_rewrite, + }; + ( + true, + claim + .deliver_large(services, &origin.identity, pinned, answer) + .await, + ) + } + Delivery::Send(response) => (false, response), + Delivery::FetchWhole(_) => { + panic!("should never ask for a bounded whole of a large source") + } + } + }) +} + +fn run( + claim: &MediaClaim, + services: &RuntimeServices, + response: Response, +) -> (bool, Response) { + run_through(claim, services, &TestOrigin::plain(), response) +} + +/// The body's bytes, or `None` where its stream failed. +fn collect(response: Response) -> (Response, Option>) { + let (parts, body) = response.into_parts(); + let bytes = block_on(async { + match body { + EdgeBody::Once(bytes) => Some(bytes.to_vec()), + EdgeBody::Stream(mut stream) => { + let mut collected = Vec::new(); + while let Some(chunk) = stream.next().await { + collected.extend_from_slice(&chunk.ok()?); + } + Some(collected) + } + } + }); + (Response::from_parts(parts, EdgeBody::empty()), bytes) +} + +fn header_of<'a>(response: &'a Response, name: &str) -> Option<&'a str> { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) +} + +fn whole_digest_header(bytes: &[u8]) -> String { + format!( + "sha-256=:{}:", + base64::engine::general_purpose::STANDARD.encode(Sha256::digest(bytes)) + ) +} + +#[test] +fn hashes_a_large_source_once_and_delivers_the_exact_pinned_bytes() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::ConfirmThenSign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + + let (large, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + + assert!( + large, + "should admit a source past the bounded limit with a strong tag" + ); + let jobs = signer.jobs(); + assert_eq!(jobs.len(), 1, "should sign once"); + assert_eq!( + jobs[0].source_sha256, + hex::encode(Sha256::digest(source())), + "should hand the signer the digest of every source byte" + ); + assert_eq!(jobs[0].size, 64, "should pin the declared length"); + assert!( + jobs[0].remaining > Duration::ZERO && jobs[0].remaining <= Duration::from_secs(60), + "should tell the signer what is left of its budget" + ); + assert_eq!(jobs[0].source_reference.as_deref(), Some(FETCHED_URL)); + assert_eq!(jobs[0].reader_reference.as_deref(), Some(READER_URL)); + assert_eq!(jobs[0].origin_host, "origin.example.com"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + body, + Some(source()), + "should deliver the origin's exact bytes" + ); + assert_eq!( + header_of(&response, "etag"), + Some(ETAG_V1), + "should keep the origin's validator" + ); + assert_eq!(header_of(&response, "content-length"), Some("64")); + assert_eq!( + header_of(&response, "link"), + Some(MANIFEST_LINK), + "should carry the manifest link" + ); + assert_eq!( + header_of(&response, "repr-digest"), + Some(whole_digest_header(&source()).as_str()), + "should describe the whole representation" + ); + let requests = origin.requests(); + for request in &requests { + assert_eq!( + request + .get(header::IF_MATCH) + .and_then(|value| value.to_str().ok()), + Some(ETAG_V1), + "should pin every origin read to the version" + ); + assert!( + request.get(header::RANGE).is_some(), + "should ask for exact ranges" + ); + } + assert_eq!( + origin.ranges(), + vec![Some((0, 63)), Some((0, 0)), Some((0, 63))], + "should hash once, confirm, then deliver every byte of the pinned version" + ); +} + +#[test] +fn answers_a_reader_range_with_the_exact_original_bytes() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::ConfirmThenSign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[(header::RANGE, "bytes=10-19")]); + + let (large, response) = run( + &claim, + &services, + serve(&source(), ETAG_V1, Some((10, 19)), None), + ); + let (response, body) = collect(response); + + assert!( + large, + "should admit a part whose total is past the bounded limit" + ); + assert_eq!( + signer.jobs()[0].source_sha256, + hex::encode(Sha256::digest(source())), + "should hash the whole version, not the part" + ); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + header_of(&response, "content-range"), + Some("bytes 10-19/64") + ); + assert_eq!(header_of(&response, "content-length"), Some("10")); + assert_eq!( + body, + Some(source()[10..20].to_vec()), + "should send the original bytes of the range" + ); + assert_eq!(header_of(&response, "link"), Some(MANIFEST_LINK)); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!( + origin.ranges(), + vec![Some((0, 63)), Some((0, 0)), Some((10, 19))], + "should hash the whole pinned version once, then read the reader's range pinned" + ); + + let (_, response) = run( + &claim, + &services, + serve(&source(), ETAG_V1, Some((10, 19)), None), + ); + let (response, body) = collect(response); + assert_eq!( + origin.whole_reads(), + 1, + "a second range should never hash again" + ); + assert_eq!(body, Some(source()[10..20].to_vec())); + assert_eq!(header_of(&response, "link"), Some(MANIFEST_LINK)); +} + +#[test] +fn a_cold_head_starts_no_load_and_a_warm_one_is_decorated() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let head = claim_for(&services, &Method::HEAD, &[]); + + let (large, response) = run(&head, &services, head_answer(&source(), ETAG_V1)); + let (response, body) = collect(response); + assert!(large, "should admit a HEAD of a large source"); + assert!( + origin.requests().is_empty(), + "a cold HEAD should never read the origin again" + ); + assert_eq!( + signer.attempts(), + 0, + "a cold HEAD should never call the signer" + ); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(header_of(&response, "content-length"), Some("64")); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!(header_of(&response, "link"), None); + assert_eq!(body, Some(Vec::new()), "a HEAD carries no body"); + + let get = claim_for(&services, &Method::GET, &[]); + let _ = collect(run(&get, &services, whole()).1); + let reads = origin.requests().len(); + let (_, response) = run(&head, &services, head_answer(&source(), ETAG_V1)); + let (response, body) = collect(response); + assert_eq!( + header_of(&response, "link"), + Some(MANIFEST_LINK), + "a warm HEAD carries the manifest" + ); + assert_eq!( + header_of(&response, "repr-digest"), + Some(whole_digest_header(&source()).as_str()) + ); + assert_eq!(header_of(&response, "content-length"), Some("64")); + assert_eq!(body, Some(Vec::new())); + assert_eq!( + origin.requests().len(), + reads, + "a warm HEAD should read nothing" + ); + assert_eq!(signer.attempts(), 1); +} + +#[test] +fn only_an_origin_that_says_it_serves_byte_ranges_starts_a_load() { + for accept_ranges in [None, Some("none")] { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let mut answer = whole(); + answer.headers_mut().remove(header::ACCEPT_RANGES); + if let Some(value) = accept_ranges { + answer + .headers_mut() + .insert(header::ACCEPT_RANGES, HeaderValue::from_static(value)); + } + + let (_, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + + assert!( + origin.requests().is_empty(), + "{accept_ranges:?}: should never hash" + ); + assert_eq!(signer.attempts(), 0, "{accept_ranges:?}"); + assert_eq!( + body, + Some(source()), + "{accept_ranges:?}: should send the origin's answer" + ); + assert_eq!( + header_of(&response, "etag"), + Some(ETAG_V1), + "{accept_ranges:?}" + ); + assert_eq!(header_of(&response, "link"), None, "{accept_ranges:?}"); + } + + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let _ = collect(run(&claim, &services, whole()).1); + let mut answer = whole(); + answer.headers_mut().remove(header::ACCEPT_RANGES); + let (_, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + assert_eq!(body, Some(source())); + assert_eq!( + header_of(&response, "link"), + Some(MANIFEST_LINK), + "a kept signed outcome still decorates the version" + ); + assert_eq!(signer.attempts(), 1); +} + +#[test] +fn a_kept_outcome_answers_without_hashing_and_never_outlives_the_policy() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let _ = collect(run(&claim, &services, whole()).1); + let reads = origin.requests().len(); + + let (_, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + assert_eq!( + signer.attempts(), + 1, + "should reuse the outcome kept for the version" + ); + assert_eq!( + origin.requests().len(), + reads, + "should send the origin's own answer" + ); + assert_eq!(body, Some(source())); + assert_eq!(header_of(&response, "link"), Some(MANIFEST_LINK)); + + let (_, revalidated) = run(&claim, &services, not_modified(ETAG_V1)); + assert_eq!(revalidated.status(), StatusCode::NOT_MODIFIED); + assert_eq!( + header_of(&revalidated, "link"), + Some(MANIFEST_LINK), + "should tell a revalidating reader of the manifest" + ); + + signer.set_namespace(Ok("policy-2".to_owned())); + let _ = collect(run(&claim, &services, whole()).1); + assert_eq!( + signer.attempts(), + 2, + "a changed policy should never find the old outcome" + ); + + signer.set_namespace(Err(MediaSigningFailure { reason: "revoked" })); + let reads = origin.requests().len(); + let (_, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + assert_eq!(signer.attempts(), 2, "a revoked policy should sign nothing"); + assert_eq!(origin.requests().len(), reads, "should hash nothing"); + assert_eq!( + header_of(&response, "link"), + None, + "should carry no manifest" + ); + assert_eq!( + header_of(&response, "etag"), + Some(ETAG_V1), + "should keep the origin's validator on its unchanged bytes" + ); + assert_eq!(body, Some(source())); +} + +#[test] +fn a_refused_source_is_hashed_once_and_then_sent_as_it_came() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[(header::RANGE, "bytes=0-9")]); + let forwarding = TestOrigin::forwarding(&[(header::RANGE, "bytes=0-9")]); + + let (_, response) = run_through( + &claim, + &services, + &forwarding, + serve(&source(), ETAG_V1, Some((0, 9)), None), + ); + let (response, body) = collect(response); + assert_eq!(signer.attempts(), 1); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(header_of(&response, "content-range"), Some("bytes 0-9/64")); + assert_eq!(body, Some(source()[0..10].to_vec())); + assert_eq!(header_of(&response, "link"), None); + assert_eq!( + header_of(&response, "etag"), + Some(ETAG_V1), + "should keep the origin's validator" + ); + assert_eq!( + header_of(&response, "cache-control"), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "a fresh fallback is held by no cache, even for the same version" + ); + let fresh = origin + .requests() + .last() + .cloned() + .expect("should ask the origin afresh"); + assert_eq!( + requested_range(&fresh), + Some((0, 9)), + "should ask afresh exactly as the reader asked" + ); + assert!( + fresh.get(header::IF_MATCH).is_none(), + "an unsigned answer is not pinned" + ); + let reads = origin.requests().len(); + + for _ in 0..3 { + let (_, response) = run_through( + &claim, + &services, + &forwarding, + serve(&source(), ETAG_V1, Some((0, 9)), None), + ); + let (response, body) = collect(response); + assert_eq!(body, Some(source()[0..10].to_vec())); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!(header_of(&response, "link"), None); + } + assert_eq!( + signer.attempts(), + 1, + "a kept refusal should never call the signer again" + ); + assert_eq!( + origin.requests().len(), + reads, + "a kept refusal should never read the origin again" + ); + assert_eq!(origin.whole_reads(), 1, "should hash once in all"); +} + +#[test] +fn a_declined_signed_origin_is_asked_afresh_with_a_fresh_signature() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[(header::RANGE, "bytes=0-9")]); + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); + headers.insert( + header::REFERER, + HeaderValue::from_static("https://www.example.com/page"), + ); + headers.insert("x-test-origin-key", HeaderValue::from_static("from-reader")); + let signed = TestOrigin { + headers, + auth: Some(OriginSigning::new(Arc::new(TestOriginSigner), None)), + unsigned_headers: vec![header::REFERER], + ..TestOrigin::plain() + }; + + let (_, response) = run_through( + &claim, + &services, + &signed, + serve(&source(), ETAG_V1, Some((0, 9)), None), + ); + let (response, body) = collect(response); + + assert_eq!(body, Some(source()[0..10].to_vec())); + assert_eq!(header_of(&response, "link"), None); + let fresh = origin + .requests() + .last() + .cloned() + .expect("should ask the origin afresh"); + assert_eq!( + fresh + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some(ORIGIN_SIGNATURE), + "should sign the fresh request afresh" + ); + assert_eq!(requested_range(&fresh), Some((0, 9))); + assert!(fresh.get(header::REFERER).is_none()); + assert!(fresh.get("x-test-origin-key").is_none()); + assert!(fresh.get(header::IF_MATCH).is_none()); +} + +#[test] +fn a_not_modified_pending_namespace_obeys_the_large_budget() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let mut signer = LargeSigner::new(LargeBehavior::Sign); + let (namespace_ready, namespace_gate) = oneshot::channel(); + let configured = Arc::get_mut(&mut signer).expect("should exclusively own the signer"); + configured.large_duration = Duration::from_secs(5); + *configured + .namespace_gate + .lock() + .expect("should lock namespace gate") = Some(namespace_gate); + let timer = Arc::new(ElapsedTimer::default()); + let cache = Arc::new(PendingLargeCache::default()); + let services = services_with_timer( + &signer, + &origin, + Arc::clone(&timer) as Arc, + ) + .with_loading_cache(Arc::clone(&cache) as Arc); + let claim = claim_for(&services, &Method::GET, &[]); + let response = not_modified(ETAG_V1); + let headers = response.headers().clone(); + let facts = OriginFacts::of(response.headers()); + + let delivery = claim + .deliver(&services, &identity(), Answered::Origin(facts), response) + .now_or_never() + .expect("should stop waiting for the namespace at the large deadline"); + + assert_unchanged_not_modified(delivery, &headers); + assert_eq!(*cache.calls.lock().expect("should lock cache calls"), 0); + assert!(origin.requests().is_empty()); + assert_eq!(signer.attempts(), 0); + let requested = timer.requested.lock().expect("should lock timer requests"); + assert_eq!(requested.len(), 1); + assert!(requested[0] > Duration::ZERO && requested[0] <= Duration::from_secs(5)); + drop(namespace_ready); +} + +#[test] +fn a_not_modified_pending_cache_obeys_the_large_budget() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let mut signer = LargeSigner::new(LargeBehavior::Sign); + Arc::get_mut(&mut signer) + .expect("should exclusively own the signer") + .large_duration = Duration::from_secs(5); + let timer = Arc::new(ElapsedTimer::default()); + let cache = Arc::new(PendingLargeCache::default()); + let services = services_with_timer( + &signer, + &origin, + Arc::clone(&timer) as Arc, + ) + .with_loading_cache(Arc::clone(&cache) as Arc); + let claim = claim_for(&services, &Method::GET, &[]); + let response = not_modified(ETAG_V1); + let headers = response.headers().clone(); + let facts = OriginFacts::of(response.headers()); + + let delivery = claim + .deliver(&services, &identity(), Answered::Origin(facts), response) + .now_or_never() + .expect("should stop waiting for the cached outcome at the large deadline"); + + assert_unchanged_not_modified(delivery, &headers); + assert_eq!(*cache.calls.lock().expect("should lock cache calls"), 1); + assert!(origin.requests().is_empty()); + assert_eq!(signer.attempts(), 0); + let requested = timer.requested.lock().expect("should lock timer requests"); + assert_eq!(requested.len(), 1); + assert!(requested[0] > Duration::ZERO && requested[0] <= Duration::from_secs(5)); +} + +#[test] +fn a_not_modified_without_a_timer_or_positive_large_budget_starts_no_io() { + for (has_timer, budget) in [(false, Duration::from_secs(5)), (true, Duration::ZERO)] { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let mut signer = LargeSigner::new(LargeBehavior::Sign); + let (namespace_ready, namespace_gate) = oneshot::channel(); + let configured = Arc::get_mut(&mut signer).expect("should exclusively own the signer"); + configured.large_duration = budget; + *configured + .namespace_gate + .lock() + .expect("should lock namespace gate") = Some(namespace_gate); + let cache = Arc::new(PendingLargeCache::default()); + let services = + build_services_with_http_client(Arc::clone(&origin) as Arc) + .with_media_signer(Some(Arc::clone(&signer) as Arc)) + .with_loading_cache(Arc::clone(&cache) as Arc); + let services = if has_timer { + services.with_timer(Arc::new(ElapsedTimer::default())) + } else { + services + }; + let claim = claim_for(&services, &Method::GET, &[]); + let response = not_modified(ETAG_V1); + let headers = response.headers().clone(); + let facts = OriginFacts::of(response.headers()); + + let delivery = claim + .deliver(&services, &identity(), Answered::Origin(facts), response) + .now_or_never() + .expect("should refuse before polling namespace without a timer and positive budget"); + + assert_unchanged_not_modified(delivery, &headers); + assert!( + signer + .namespace_gate + .lock() + .expect("should lock namespace gate") + .is_some(), + "should never call namespace resolution" + ); + assert_eq!(*cache.calls.lock().expect("should lock cache calls"), 0); + assert!(origin.requests().is_empty()); + assert_eq!(signer.attempts(), 0); + drop(namespace_ready); + } +} + +#[test] +fn a_namespace_pending_past_the_large_budget_starts_no_hash_or_signing() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let mut signer = LargeSigner::new(LargeBehavior::Sign); + let (namespace_ready, namespace_gate) = oneshot::channel(); + let configured = Arc::get_mut(&mut signer).expect("should exclusively own the signer"); + configured.large_duration = Duration::from_secs(5); + *configured + .namespace_gate + .lock() + .expect("should lock namespace gate") = Some(namespace_gate); + let timer = Arc::new(ElapsedTimer::default()); + let services = services_with_timer( + &signer, + &origin, + Arc::clone(&timer) as Arc, + ); + let claim = claim_for(&services, &Method::GET, &[]); + let response = whole(); + let facts = OriginFacts::of(response.headers()); + let Delivery::Large(answer) = + block_on(claim.deliver(&services, &identity(), Answered::Origin(facts), response)) + else { + panic!("should admit the source to the large lane"); + }; + let url = url::Url::parse(FETCHED_URL).expect("should parse the fetched address"); + let headers = HeaderMap::new(); + let pinned = PinnedOrigin { + url: &url, + headers: &headers, + backend_name: "origin", + auth: None, + unsigned_headers: &[], + response_header_rewrite: None, + }; + + // Keep the policy future unresolved while the deterministic large timer + // expires. A missing namespace race must fail here, never hang the test. + let response = claim + .deliver_large(&services, &identity(), pinned, answer) + .now_or_never() + .expect("should return unsigned when the large budget expires during namespace resolution"); + let (response, body) = collect(response); + + assert_eq!(body, Some(source()), "should preserve the original body"); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!(header_of(&response, "link"), None); + assert_eq!(header_of(&response, "repr-digest"), None); + assert!(origin.requests().is_empty(), "should never start hashing"); + assert_eq!(signer.attempts(), 0, "should never start signing"); + assert!(signer.jobs().is_empty()); + let requested = timer.requested.lock().expect("should lock timer requests"); + assert_eq!(requested.len(), 1, "should race the namespace once"); + assert!( + requested[0] > Duration::ZERO && requested[0] <= Duration::from_secs(5), + "should use the large budget rather than the policy's own hold limit" + ); + drop(namespace_ready); +} + +#[test] +fn an_exhausted_large_budget_starts_no_hash_or_signing() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_with_timer(&signer, &origin, Arc::new(ElapsedTimer::default())); + let load = LargeLoad { + signer: Arc::clone(&signer) as Arc, + services, + url: url::Url::parse(FETCHED_URL).expect("should parse the fetched address"), + headers: HeaderMap::new(), + backend_name: "origin".to_owned(), + auth: None, + unsigned_headers: Vec::new(), + version: SourceVersion::new(ETAG_V1, to_u64(source().len())) + .expect("should pin the test version"), + reader_url: READER_URL.to_owned(), + content_type: "image/png".to_owned(), + references: LargeReferences::of(&identity(), false), + namespace: "policy-1".to_owned(), + budget: LargeBudget { + started: Instant::now(), + total: Duration::ZERO, + }, + keep_for: Duration::from_secs(600), + }; + + let loaded = block_on(load.run()); + + assert!( + matches!( + decode_kept_outcome(&loaded.bytes), + Some(KeptOutcome::Declined) + ), + "should keep an unsigned outcome" + ); + assert!(origin.requests().is_empty(), "should never start hashing"); + assert_eq!(signer.attempts(), 0, "should never start signing"); +} + +#[test] +fn a_timed_out_source_is_hashed_once_and_then_sent_as_it_came() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Hang); + let services = services_with_timer(&signer, &origin, Arc::new(ElapsedTimer::default())); + let claim = claim_for(&services, &Method::GET, &[]); + + let (large, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + assert!(large); + assert!( + signer.jobs().is_empty(), + "a signer out of time signs nothing" + ); + assert_eq!(body, Some(source())); + assert_eq!(header_of(&response, "link"), None); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + let reads = origin.requests().len(); + + let (_, response) = run(&claim, &services, whole()); + let (_, body) = collect(response); + assert_eq!(body, Some(source())); + assert_eq!( + signer.attempts(), + 1, + "a kept timeout should never call the signer again" + ); + assert_eq!( + origin.requests().len(), + reads, + "a kept timeout should never hash again" + ); +} + +#[test] +fn a_version_change_before_publication_signs_nothing_and_sends_the_current_version() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + // The hash reads the first version; the signer's confirmation and the + // fresh answer meet the second. + origin.flip_after(1, &other_source(), ETAG_V2); + let signer = LargeSigner::new(LargeBehavior::ConfirmThenSign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + + let (large, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + + assert!(large); + assert!(signer.jobs().is_empty(), "should publish nothing"); + assert_eq!(signer.failures(), vec!["source_version_changed"]); + assert_eq!( + response.status(), + StatusCode::OK, + "an unsigned answer is the origin's, not a refusal" + ); + assert_eq!( + body, + Some(other_source()), + "should send the version the origin now serves" + ); + assert_eq!( + header_of(&response, "etag"), + Some(ETAG_V2), + "should describe the bytes sent by their own validator" + ); + assert_eq!(header_of(&response, "link"), None); + assert_eq!(header_of(&response, "repr-digest"), None); +} + +#[test] +fn a_fresh_fallback_is_never_cached_and_keeps_both_security_policies() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let caller_policy = "img-src 'self'"; + let fresh_policy = "default-src 'none'"; + // The same version answers afresh with a shorter lifetime and a + // security policy of its own. + origin.flip_after_with( + 1, + &source(), + ETAG_V1, + vec![ + ( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=5"), + ), + ( + header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static(fresh_policy), + ), + ], + ); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let mut answer = whole(); + answer.headers_mut().insert( + header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static(caller_policy), + ); + + let (_, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + + assert_eq!(body, Some(source())); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!( + header_of(&response, "cache-control"), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "should keep neither the first answer's lifetime nor the fresh one's" + ); + let policies: Vec<_> = response + .headers() + .get_all(header::CONTENT_SECURITY_POLICY) + .iter() + .collect(); + assert_eq!( + policies, + vec![caller_policy, fresh_policy], + "should add the fresh policy to the caller's, never replace it" + ); +} + +#[test] +fn a_refused_source_that_changed_keeps_the_new_versions_provenance_and_privacy() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let origin_link = "; rel=\"c2pa-manifest\""; + // The hash reads the first version; the fresh answer meets a second + // that names a manifest of its own and may be held by no shared cache. + origin.flip_after_with( + 1, + &other_source(), + ETAG_V2, + vec![ + ( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ), + (header::LINK, HeaderValue::from_static(origin_link)), + ( + header::SET_COOKIE, + HeaderValue::from_static("session=reader-1"), + ), + ], + ); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let mut answer = whole(); + answer.headers_mut().insert( + header::EXPIRES, + HeaderValue::from_static("Wed, 21 Oct 2026 07:28:00 GMT"), + ); + answer + .headers_mut() + .insert(header::AGE, HeaderValue::from_static("30")); + + let (_, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + + assert_eq!(signer.attempts(), 1); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + body, + Some(other_source()), + "should send the version the origin now serves" + ); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V2)); + let links: Vec<_> = response.headers().get_all(header::LINK).iter().collect(); + assert_eq!( + links, + vec![origin_link], + "should keep the new version's manifest link" + ); + assert_eq!( + header_of(&response, "cache-control"), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "should never send the first version's shared cache policy with other bytes" + ); + assert_eq!(header_of(&response, "expires"), None); + assert_eq!(header_of(&response, "age"), None); + assert_eq!( + header_of(&response, "set-cookie"), + None, + "should pass on no cookie the caller's handling never saw" + ); + assert_eq!(header_of(&response, "repr-digest"), None); +} + +#[test] +fn a_fresh_fallback_keeps_redirect_challenge_and_retry_response_semantics() { + for (status, name, value) in [ + ( + StatusCode::TEMPORARY_REDIRECT, + header::LOCATION, + "/media/moved", + ), + ( + StatusCode::UNAUTHORIZED, + header::WWW_AUTHENTICATE, + "Bearer realm=\"media\"", + ), + (StatusCode::SERVICE_UNAVAILABLE, header::RETRY_AFTER, "60"), + ] { + let origin = RangeOrigin::new(&source(), ETAG_V1); + origin.flip_after_with( + 1, + &source(), + ETAG_V1, + vec![(name.clone(), HeaderValue::from_static(value))], + ); + origin + .state + .lock() + .expect("should lock origin") + .status_after = Some((1, status)); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let (_, response) = run(&claim, &services, whole()); + let (response, _) = collect(response); + + assert_eq!(response.status(), status); + assert_eq!( + response.headers().get(&name), + Some(&HeaderValue::from_static(value)) + ); + assert_eq!( + header_of(&response, "cache-control"), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + ); + assert_eq!(signer.attempts(), 1); + assert_eq!( + origin.requests().len(), + 2, + "should not follow or retry the fresh response" + ); + } +} + +#[test] +fn a_fresh_publisher_redirect_uses_the_existing_origin_url_mapping_without_discarded_cookies() { + for (location, expected) in [ + ( + "https://origin.example.com/media/moved", + "https://www.example.com/media/moved", + ), + (FETCHED_URL, FETCHED_URL), + ] { + let origin = RangeOrigin::new(&source(), ETAG_V1); + origin.flip_after_with( + 1, + &source(), + ETAG_V1, + vec![ + (header::LOCATION, HeaderValue::from_static(location)), + ( + header::SET_COOKIE, + HeaderValue::from_static("session=new; Path=/"), + ), + ( + header::CONTENT_LOCATION, + HeaderValue::from_static("https://origin.example.com/media/fresh"), + ), + ], + ); + origin + .state + .lock() + .expect("should lock origin") + .status_after = Some((1, StatusCode::TEMPORARY_REDIRECT)); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let publisher = TestOrigin { + response_header_rewrite: Some(crate::response_header_rewrite::OriginHeaderRewrite { + origin_host: "origin.example.com", + origin_host_header: "origin.example.com", + origin_path: "", + request_host: "www.example.com", + request_scheme: "https", + origin_scheme: "https", + request_path_and_query: PATH, + request_is_get_or_head: true, + }), + ..TestOrigin::plain() + }; + let mut initial = whole(); + initial + .headers_mut() + .insert(header::CONTENT_LOCATION, HeaderValue::from_static("/stale")); + let (_, response) = run_through(&claim, &services, &publisher, initial); + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + assert_eq!( + response.headers().get(header::LOCATION), + Some(&HeaderValue::from_static(expected)), + "a discarded cookie must not justify redirecting to the same serving URL", + ); + assert!(!response.headers().contains_key(header::SET_COOKIE)); + assert_eq!( + response.headers().get(header::CONTENT_LOCATION), + Some(&HeaderValue::from_static( + "https://www.example.com/media/fresh" + )), + ); + } +} + +#[test] +fn a_version_change_after_signing_is_refused_rather_than_mixed() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + // The hash reads the first version; the signed delivery meets the + // second. + origin.flip_after(1, &other_source(), ETAG_V2); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + + let (_, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + + assert_eq!(signer.jobs().len(), 1); + assert_eq!( + response.status(), + StatusCode::SERVICE_UNAVAILABLE, + "should never send other bytes under the signed version's headers" + ); + assert_eq!(header_of(&response, "link"), None); + assert_eq!(body, Some(Vec::new())); +} + +#[test] +fn a_cut_hash_read_signs_nothing_and_is_never_resumed() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + origin.cut_next_at(23); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + + let (large, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + + assert!(large); + assert_eq!(signer.attempts(), 0, "should sign nothing from a cut read"); + assert_eq!( + origin.ranges(), + vec![Some((0, 63)), None], + "should neither resume nor restart the hash, only ask the origin afresh once as the reader asked" + ); + assert_eq!( + body, + Some(source()), + "should deliver the origin's fresh answer" + ); + assert_eq!( + header_of(&response, "link"), + None, + "should carry no manifest" + ); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + + let (_, response) = run(&claim, &services, whole()); + let (_, body) = collect(response); + assert_eq!(body, Some(source())); + assert_eq!( + origin.requests().len(), + 2, + "a kept failure should never hash again" + ); + + let origin = RangeOrigin::new(&source(), ETAG_V1); + origin.cut_next_at(23); + origin.flip_after(1, &other_source(), ETAG_V2); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let (_, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + assert_eq!(signer.attempts(), 0); + assert_eq!( + response.status(), + StatusCode::OK, + "an unsigned answer is the origin's, not a refusal" + ); + assert_eq!(body, Some(other_source())); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V2)); + assert_eq!(header_of(&response, "link"), None); +} + +#[test] +fn a_source_no_public_address_names_is_never_hashed() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let queried = TestOrigin { + identity: VariantIdentity { + fetched_url: QUERY_FETCHED_URL, + reader_query_free: false, + ..identity() + }, + ..TestOrigin::plain() + }; + + for _ in 0..2 { + let (_, response) = run_through(&claim, &services, &queried, whole()); + let (response, body) = collect(response); + assert_eq!( + body, + Some(source()), + "should send the origin's answer as it came" + ); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!(header_of(&response, "link"), None); + } + assert!( + origin.requests().is_empty(), + "should never read a source no public address names" + ); + assert_eq!(signer.attempts(), 0); + + let reader_named = TestOrigin { + identity: VariantIdentity { + fetched_url: QUERY_FETCHED_URL, + ..identity() + }, + ..TestOrigin::plain() + }; + let _ = collect(run_through(&claim, &services, &reader_named, whole()).1); + let jobs = signer.jobs(); + assert_eq!( + jobs[0].source_reference, None, + "should never name a fetched address with a query" + ); + assert_eq!( + jobs[0].reader_reference.as_deref(), + Some(READER_URL), + "should name the reader's query-free address" + ); +} + +#[test] +fn a_signed_origin_address_is_never_named_and_its_reads_carry_no_reader_header() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::ConfirmThenSign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let mut headers = HeaderMap::new(); + headers.insert(header::HOST, HeaderValue::from_static("origin.example.com")); + headers.insert( + header::REFERER, + HeaderValue::from_static("https://www.example.com/page"), + ); + headers.insert( + header::IF_NONE_MATCH, + HeaderValue::from_static("\"reader\""), + ); + headers.insert("x-test-origin-key", HeaderValue::from_static("from-reader")); + let signed = TestOrigin { + headers, + auth: Some(OriginSigning::new(Arc::new(TestOriginSigner), None)), + unsigned_headers: vec![header::REFERER], + ..TestOrigin::plain() + }; + + let (_, response) = run_through(&claim, &services, &signed, whole()); + let (response, body) = collect(response); + + let jobs = signer.jobs(); + assert_eq!( + jobs[0].source_reference, None, + "should never name an address core signs" + ); + assert_eq!(jobs[0].reader_reference.as_deref(), Some(READER_URL)); + assert_eq!(body, Some(source())); + assert_eq!(header_of(&response, "link"), Some(MANIFEST_LINK)); + let requests = origin.requests(); + assert_eq!(requests.len(), 3, "should hash, confirm and deliver"); + for request in &requests { + assert_eq!( + request + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some(ORIGIN_SIGNATURE), + "should sign every pinned read afresh" + ); + assert!( + request.get(header::REFERER).is_none(), + "should send no reader header a signed read must not carry" + ); + assert!( + request.get(header::IF_NONE_MATCH).is_none(), + "should send none of the reader's conditions" + ); + assert!( + request.get("x-test-origin-key").is_none(), + "should send no header in the signer's namespace" + ); + assert_eq!( + request + .get(header::IF_MATCH) + .and_then(|value| value.to_str().ok()), + Some(ETAG_V1) + ); + } + + let queried_reader = TestOrigin { + identity: VariantIdentity { + reader_query_free: false, + ..identity() + }, + auth: Some(OriginSigning::new(Arc::new(TestOriginSigner), None)), + ..TestOrigin::plain() + }; + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let _ = collect(run_through(&claim, &services, &queried_reader, whole()).1); + assert!( + origin.requests().is_empty(), + "should decline before hashing when only the signed address could name the source" + ); + assert_eq!(signer.attempts(), 0); +} + +#[test] +fn an_answer_whose_outcome_cannot_be_kept_is_never_hashed() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + for cache_control in ["no-cache", "public"] { + let mut answer = whole(); + answer.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static(cache_control), + ); + let (large, response) = run(&claim, &services, answer); + let (_, body) = collect(response); + assert!(!large, "{cache_control}: should not enter the large lane"); + assert_eq!(body, Some(source())); + } + + let keeps_nothing = + build_services_with_http_client(Arc::clone(&origin) as Arc) + .with_media_signer(Some(Arc::clone(&signer) as Arc)) + .with_timer(Arc::new(StoppedTimer)); + let claim = claim_for(&keeps_nothing, &Method::GET, &[]); + let (large, response) = run(&claim, &keeps_nothing, whole()); + let (_, body) = collect(response); + assert!( + !large, + "should not enter the large lane where nothing is kept" + ); + assert_eq!(body, Some(source())); + + assert!( + origin.requests().is_empty(), + "should never read the origin again" + ); + assert_eq!(signer.attempts(), 0); +} + +#[test] +fn an_origin_manifest_link_is_kept_rather_than_competed_with() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let origin_link = "; rel=\"c2pa-manifest\""; + let mut answer = whole(); + answer + .headers_mut() + .insert(header::LINK, HeaderValue::from_static(origin_link)); + + let (large, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + + assert!( + !large, + "should not sign a source that names its own manifest" + ); + assert_eq!(body, Some(source())); + let links: Vec<_> = response.headers().get_all(header::LINK).iter().collect(); + assert_eq!( + links, + vec![origin_link], + "should keep only the origin's manifest link" + ); + assert!(origin.requests().is_empty()); + assert_eq!(signer.attempts(), 0); +} + +#[test] +fn a_warm_not_modified_naming_an_origin_manifest_gets_no_kept_outcome() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let _ = collect(run(&claim, &services, whole()).1); + assert_eq!( + signer.jobs().len(), + 1, + "should keep a signed outcome for the version" + ); + + let (_, plain) = run(&claim, &services, not_modified(ETAG_V1)); + assert_eq!( + header_of(&plain, "link"), + Some(MANIFEST_LINK), + "the kept outcome decorates a 304 without a link of the origin's" + ); + + let origin_link = "; rel=\"c2pa-manifest\""; + let mut linked = not_modified(ETAG_V1); + linked + .headers_mut() + .insert(header::LINK, HeaderValue::from_static(origin_link)); + let (_, response) = run(&claim, &services, linked); + + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + let links: Vec<_> = response.headers().get_all(header::LINK).iter().collect(); + assert_eq!( + links, + vec![origin_link], + "should keep the origin's manifest link alone" + ); + assert_eq!(header_of(&response, "repr-digest"), None); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!(signer.attempts(), 1, "should never sign again"); +} + +#[test] +fn a_source_without_a_strong_tag_is_never_given_a_version() { + for etag in ["W/\"v1\"", ""] { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let mut answer = whole(); + if etag.is_empty() { + answer.headers_mut().remove(header::ETAG); + } else { + answer + .headers_mut() + .insert(header::ETAG, HeaderValue::from_static("W/\"v1\"")); + } + + let (large, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + + assert!(!large, "{etag:?}: should not enter the large lane"); + assert_eq!(signer.attempts(), 0); + assert!( + origin.requests().is_empty(), + "{etag:?}: should not ask the origin again" + ); + assert_eq!( + body, + Some(source()), + "{etag:?}: should send the origin's answer as it came" + ); + assert_eq!(header_of(&response, "link"), None); + } +} + +#[test] +fn a_pinned_read_refuses_another_version_or_range() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let url = url::Url::parse(FETCHED_URL).expect("should parse the fetched address"); + let headers = HeaderMap::new(); + let version = SourceVersion::new(ETAG_V1, 64).expect("should take a strong tag"); + let reader = VersionedMediaSource::new(&services, &url, &headers, "origin", version); + + let mut part = [0_u8; 8]; + block_on(reader.read_exact_at(8, &mut part)).expect("should read the pinned range"); + assert_eq!(part.to_vec(), source()[8..16].to_vec()); + assert_eq!( + block_on(reader.read_exact_at(60, &mut part)), + Err(MediaSourceError::OutOfBounds), + "should refuse a read past the version before asking" + ); + + origin.answer_wrong_content_range(); + assert_eq!( + block_on(reader.read_exact_at(8, &mut part)), + Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_content_range" + }) + ); + + origin.flip_after(0, &other_source(), ETAG_V2); + assert_eq!( + block_on(reader.confirm_current()), + Err(MediaSourceError::VersionChanged) + ); + assert!( + SourceVersion::new("W/\"v1\"", 64).is_none(), + "a weak tag names no version" + ); +} diff --git a/crates/trusted-server-core/src/media_signing/test_support.rs b/crates/trusted-server-core/src/media_signing/test_support.rs new file mode 100644 index 0000000000..f6e5ddb075 --- /dev/null +++ b/crates/trusted-server-core/src/media_signing/test_support.rs @@ -0,0 +1,164 @@ +//! A [`MediaSigner`] for core's tests, which records each job and appends a +//! fixed suffix to the source in place of a manifest. + +use std::sync::{Arc, Mutex}; + +use bytes::Bytes; +use http::{HeaderName, HeaderValue, header}; + +use super::{ + MediaJob, MediaNamespacing, MediaSigner, MediaSigning, MediaSigningFailure, SignedMedia, +}; +use crate::platform::RuntimeServices; + +/// What the stub's signing appends to the source. +pub(crate) const SIGNED_SUFFIX: &[u8] = b"|signed-manifest"; + +/// The header the stub writes on a signed representation. +pub(crate) const MANIFEST_HEADER: &str = "x-test-manifest"; + +/// The `Link` the stub's [`Behavior::SignWithLink`] writes. +pub(crate) const MANIFEST_LINK: &str = "; rel=\"c2pa-manifest\""; + +/// How the stub answers a job. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Behavior { + /// Returns the source with [`SIGNED_SUFFIX`] appended. + Sign, + /// Returns a failure. + Fail, + /// Returns signed bytes with a header core owns. + ReservedHeader, + /// Signs as [`Behavior::Sign`] and adds a manifest [`MANIFEST_LINK`]. + SignWithLink, +} + +/// One job the stub was handed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RecordedJob { + pub(crate) reader_url: String, + pub(crate) content_type: String, + pub(crate) source: Vec, + pub(crate) source_sha256: String, + pub(crate) source_reference: Option, + pub(crate) source_origin_host: Option, + pub(crate) optimizer_output: bool, +} + +/// Leaves `/api/` paths alone and signs every `image/` type. +pub(crate) struct StubSigner { + behavior: Behavior, + limit: usize, + jobs: Mutex>, + namespace: Mutex>, +} + +impl StubSigner { + pub(crate) fn new(behavior: Behavior, limit: usize) -> Arc { + Arc::new(Self { + behavior, + limit, + jobs: Mutex::new(Vec::new()), + namespace: Mutex::new(Ok("policy-1".to_owned())), + }) + } + + /// Makes [`MediaSigner::namespace`] answer `namespace` from now on, as a + /// changed or revoked policy would. + pub(crate) fn set_namespace(&self, namespace: Result) { + *self.namespace.lock().expect("should lock namespace") = namespace; + } + + pub(crate) fn jobs(&self) -> Vec { + self.jobs.lock().expect("should lock jobs").clone() + } + + /// `services` carrying this signer, as an adapter hands it in. + pub(crate) fn installed_in(self: &Arc, services: RuntimeServices) -> RuntimeServices { + services.with_media_signer(Some(Arc::clone(self) as Arc)) + } +} + +impl MediaSigner for StubSigner { + fn signer_id(&self) -> &'static str { + "testing.media" + } + + fn handles_path(&self, path: &str) -> bool { + !path.starts_with("/api/") + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type.starts_with("image/") + } + + fn max_source_bytes(&self) -> usize { + self.limit + } + + fn namespace<'a>(&'a self, _services: &'a RuntimeServices) -> MediaNamespacing<'a> { + let namespace = self + .namespace + .lock() + .expect("should lock namespace") + .clone(); + Box::pin(async move { namespace }) + } + + fn sign<'a>( + &'a self, + _services: &'a RuntimeServices, + job: &'a MediaJob<'a>, + ) -> MediaSigning<'a> { + Box::pin(async move { + self.jobs + .lock() + .expect("should lock jobs") + .push(RecordedJob { + reader_url: job.reader_url.to_owned(), + content_type: job.content_type.to_owned(), + source: job.source.to_vec(), + source_sha256: job.source_sha256.to_owned(), + source_reference: job.source_reference.map(str::to_owned), + source_origin_host: job.source_origin_host.map(str::to_owned), + optimizer_output: job.optimizer_output, + }); + // Signs under the namespace current when it signs, as a real + // signer reports the policy it used. + let namespace = self + .namespace + .lock() + .expect("should lock namespace") + .clone() + .unwrap_or_default(); + let mut bytes = job.source.to_vec(); + bytes.extend_from_slice(SIGNED_SUFFIX); + match self.behavior { + Behavior::Fail => Err(MediaSigningFailure { + reason: "stub_failure", + }), + Behavior::Sign => Ok(SignedMedia { + bytes: Bytes::from(bytes), + content_type: HeaderValue::from_static("image/png"), + headers: vec![( + HeaderName::from_static(MANIFEST_HEADER), + HeaderValue::from_static("instance-1"), + )], + namespace, + }), + Behavior::SignWithLink => Ok(SignedMedia { + bytes: Bytes::from(bytes), + content_type: HeaderValue::from_static("image/png"), + headers: vec![(header::LINK, HeaderValue::from_static(MANIFEST_LINK))], + namespace, + }), + Behavior::ReservedHeader => Ok(SignedMedia { + bytes: Bytes::from(bytes), + content_type: HeaderValue::from_static("image/png"), + headers: vec![(header::ETAG, HeaderValue::from_static("\"forged\""))], + namespace, + }), + } + }) + } +} diff --git a/crates/trusted-server-core/src/media_signing/tests.rs b/crates/trusted-server-core/src/media_signing/tests.rs new file mode 100644 index 0000000000..da92c92291 --- /dev/null +++ b/crates/trusted-server-core/src/media_signing/tests.rs @@ -0,0 +1,1544 @@ +use std::sync::Arc; + +use base64::Engine as _; +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use futures::StreamExt as _; +use http::{HeaderMap, HeaderName, HeaderValue, Method, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; + +use super::test_support::{Behavior, MANIFEST_LINK, SIGNED_SUFFIX, StubSigner}; +use super::*; +use crate::platform::test_support::{MemoryLoadingCache, noop_services}; +use crate::test_support::tests::create_test_settings; + +const SOURCE: &[u8] = b"origin-image-bytes"; +const PATH: &str = "/media/42"; +const FETCHED_URL: &str = "https://origin.example.com/media/42"; +const READER_URL: &str = "https://www.example.com/media/42"; +const ORIGIN_IDENTITY: &str = "https://origin.example.com\0origin.example.com"; +const ORIGIN_ETAG: &str = "\"origin-etag\""; + +fn identity() -> VariantIdentity<'static> { + VariantIdentity { + reader_url: READER_URL, + origin_identity: ORIGIN_IDENTITY, + fetched_url: FETCHED_URL, + reader_query_free: true, + } +} + +fn signed_bytes() -> Vec { + let mut bytes = SOURCE.to_vec(); + bytes.extend_from_slice(SIGNED_SUFFIX); + bytes +} + +fn signed_etag() -> String { + format!("\"sig-{}\"", hex::encode(Sha256::digest(signed_bytes()))) +} + +fn request_headers(pairs: &[(HeaderName, &str)]) -> HeaderMap { + let mut headers = HeaderMap::new(); + for (name, value) in pairs { + headers.append( + name.clone(), + HeaderValue::from_str(value).expect("should build header"), + ); + } + headers +} + +/// The headers of a browser asking for an image, which may be answered +/// from a kept copy. +fn image_request(pairs: &[(HeaderName, &str)]) -> HeaderMap { + let mut headers = request_headers(pairs); + headers.insert("sec-fetch-dest", HeaderValue::from_static("image")); + headers +} + +fn origin_response( + status: StatusCode, + body: EdgeBody, + extra: &[(HeaderName, &str)], +) -> Response { + let mut builder = Response::builder() + .status(status) + .header(header::CONTENT_TYPE, "image/png") + .header(header::ETAG, ORIGIN_ETAG) + .header(header::CACHE_CONTROL, "public, max-age=600"); + for (name, value) in extra { + builder = builder.header(name.clone(), *value); + } + builder.body(body).expect("should build origin response") +} + +fn whole_origin() -> Response { + origin_response(StatusCode::OK, EdgeBody::from(SOURCE.to_vec()), &[]) +} + +fn part_origin() -> Response { + origin_response( + StatusCode::PARTIAL_CONTENT, + EdgeBody::from(SOURCE[0..=3].to_vec()), + &[(header::CONTENT_RANGE, "bytes 0-3/18")], + ) +} + +async fn body_bytes(body: EdgeBody) -> Vec { + match body { + EdgeBody::Once(bytes) => bytes.to_vec(), + EdgeBody::Stream(mut stream) => { + let mut collected = Vec::new(); + while let Some(chunk) = stream.next().await { + collected.extend_from_slice(&chunk.expect("should read stream chunk")); + } + collected + } + } +} + +fn header<'a>(response: &'a Response, name: &str) -> Option<&'a str> { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) +} + +fn claim_shared( + services: &RuntimeServices, + method: &Method, + headers: &HeaderMap, + shared: bool, +) -> MediaClaim { + MediaClaim::of_request( + &create_test_settings(), + services, + method, + PATH, + headers, + shared, + ) + .expect("should claim a media path") +} + +fn claim(services: &RuntimeServices, method: &Method, headers: &HeaderMap) -> MediaClaim { + claim_shared(services, method, headers, true) +} + +fn keeping(signer: &Arc) -> RuntimeServices { + signer + .installed_in(noop_services()) + .with_loading_cache(Arc::new(MemoryLoadingCache::default())) +} + +fn split(response: Response) -> (Response, Vec) { + let (parts, body) = response.into_parts(); + let bytes = futures::executor::block_on(body_bytes(body)); + (Response::from_parts(parts, EdgeBody::empty()), bytes) +} + +/// What `deliver` decides for the origin's `response`, with the facts its +/// own headers give. +fn delivery( + claim: &MediaClaim, + services: &RuntimeServices, + response: Response, +) -> Delivery { + let facts = OriginFacts::of(response.headers()); + futures::executor::block_on(claim.deliver( + services, + &identity(), + Answered::Origin(facts), + response, + )) +} + +/// Delivers the origin's `response` to a reader whose request is `method` +/// with `headers`, expecting core to send it without a second fetch. +fn deliver( + signer: &Arc, + method: &Method, + headers: &HeaderMap, + response: Response, +) -> (Response, Vec) { + let services = signer.installed_in(noop_services()); + let claim = claim(&services, method, headers); + let Delivery::Send(response) = delivery(&claim, &services, response) else { + panic!("should send without asking the origin again"); + }; + split(response) +} + +#[test] +fn signs_the_exact_captured_bytes_and_owns_the_validators() { + let signer = StubSigner::new(Behavior::Sign, 1024); + + let (response, body) = deliver(&signer, &Method::GET, &HeaderMap::new(), whole_origin()); + + assert_eq!(response.status(), StatusCode::OK, "should answer 200"); + assert_eq!( + body, + signed_bytes(), + "should send the signer's bytes unchanged" + ); + let jobs = signer.jobs(); + assert_eq!(jobs.len(), 1, "should ask the signer once"); + assert_eq!( + jobs[0].source, SOURCE, + "should hand the signer the exact origin bytes" + ); + assert_eq!(jobs[0].source_sha256, hex::encode(Sha256::digest(SOURCE))); + assert_eq!( + jobs[0].reader_url, READER_URL, + "should name the reader's address" + ); + assert_eq!( + jobs[0].content_type, "image/png", + "should pass the origin type" + ); + assert_eq!(jobs[0].source_reference.as_deref(), Some(FETCHED_URL)); + assert_eq!( + jobs[0].source_origin_host.as_deref(), + Some("origin.example.com") + ); + assert_eq!(header(&response, "etag"), Some(signed_etag().as_str())); + assert_eq!( + header(&response, "repr-digest"), + Some( + format!( + "sha-256=:{}:", + base64::engine::general_purpose::STANDARD.encode(Sha256::digest(signed_bytes())) + ) + .as_str() + ), + "should describe the signed representation's digest" + ); + assert_eq!( + header(&response, "content-length"), + Some(signed_bytes().len().to_string().as_str()), + "should frame the signed length" + ); + assert_eq!( + header(&response, "accept-ranges"), + Some("none"), + "a signed representation nothing keeps offers no ranges" + ); + assert_eq!(header(&response, "x-test-manifest"), Some("instance-1")); + assert_eq!( + header(&response, "cache-control"), + Some("public, max-age=600") + ); +} + +#[test] +fn merges_the_signers_link_with_the_origins_links() { + let signer = StubSigner::new(Behavior::SignWithLink, 1024); + let origin_link = "; rel=preload; as=style"; + + let (response, _) = deliver( + &signer, + &Method::GET, + &HeaderMap::new(), + origin_response( + StatusCode::OK, + EdgeBody::from(SOURCE.to_vec()), + &[(header::LINK, origin_link)], + ), + ); + + let links: Vec<&str> = response + .headers() + .get_all(header::LINK) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect(); + assert_eq!( + links, + vec![origin_link, MANIFEST_LINK], + "should keep the origin's link and add the manifest's" + ); +} + +#[test] +fn names_the_origin_host_even_where_the_reference_is_withheld() { + for (fetched_url, host) in [ + ( + "https://origin.example.com/media/42?signature=secret", + "origin.example.com", + ), + ( + "https://user:pass@Origin.Example.com/media/42", + "origin.example.com", + ), + ( + "https://origin.example.com:8443/media/42#part", + "origin.example.com", + ), + ] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let claim = claim(&services, &Method::GET, &HeaderMap::new()); + let response = whole_origin(); + let facts = OriginFacts::of(response.headers()); + futures::executor::block_on(claim.deliver( + &services, + &VariantIdentity { + fetched_url, + ..identity() + }, + Answered::Origin(facts), + response, + )); + let job = &signer.jobs()[0]; + assert_eq!( + job.source_reference, None, + "should send no reference for {fetched_url}" + ); + assert_eq!( + job.source_origin_host.as_deref(), + Some(host), + "should name the host of {fetched_url}" + ); + } +} + +#[test] +fn answers_a_single_range_from_the_signed_bytes() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let total = signed_bytes().len(); + + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, "bytes=2-5")]), + whole_origin(), + ); + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + body, + signed_bytes()[2..=5], + "should send the asked bytes of the signed body" + ); + assert_eq!( + header(&response, "content-range"), + Some(format!("bytes 2-5/{total}").as_str()) + ); + assert_eq!(header(&response, "content-length"), Some("4")); +} + +#[test] +fn answers_suffix_and_open_ranges_from_the_signed_bytes() { + let total = signed_bytes().len(); + for (range, first) in [("bytes=-4", total - 4), ("bytes=10-", 10)] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, range)]), + whole_origin(), + ); + assert_eq!( + response.status(), + StatusCode::PARTIAL_CONTENT, + "{range} should be partial" + ); + assert_eq!( + body, + signed_bytes()[first..], + "{range} should send the tail" + ); + assert_eq!( + header(&response, "content-range"), + Some(format!("bytes {first}-{}/{total}", total - 1).as_str()) + ); + } +} + +#[test] +fn refuses_a_range_past_the_signed_length() { + let signer = StubSigner::new(Behavior::Sign, 1024); + + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, "bytes=1000-")]), + whole_origin(), + ); + + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert!(body.is_empty(), "should send no body with a 416"); + assert_eq!( + header(&response, "content-range"), + Some(format!("bytes */{}", signed_bytes().len()).as_str()) + ); +} + +#[test] +fn answers_several_ranges_and_malformed_ranges_with_the_whole_signed_body() { + for range in ["bytes=0-1,4-5", "bytes=5-2", "items=0-1", "bytes=+1-2"] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, range)]), + whole_origin(), + ); + assert_eq!( + response.status(), + StatusCode::OK, + "{range} should be answered whole" + ); + assert_eq!(body, signed_bytes()); + } +} + +#[test] +fn honors_if_range_only_for_the_signed_entity_tag() { + let signed = signed_etag(); + for (if_range, expected) in [ + (signed.as_str(), StatusCode::PARTIAL_CONTENT), + (ORIGIN_ETAG, StatusCode::OK), + ("Tue, 01 Sep 2026 00:00:00 GMT", StatusCode::OK), + ] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, _) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, "bytes=0-3"), (header::IF_RANGE, if_range)]), + whole_origin(), + ); + assert_eq!(response.status(), expected, "If-Range {if_range}"); + } +} + +#[test] +fn tells_a_precondition_naming_signed_bytes_from_one_naming_the_origins() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let signed = signed_etag(); + for (headers, names_signed) in [ + ( + request_headers(&[ + (header::RANGE, "bytes=0-3"), + (header::IF_RANGE, signed.as_str()), + ]), + true, + ), + ( + request_headers(&[(header::IF_MATCH, format!("\"other\", {signed}").as_str())]), + true, + ), + ( + request_headers(&[ + (header::RANGE, "bytes=0-3"), + (header::IF_RANGE, ORIGIN_ETAG), + ]), + false, + ), + ( + request_headers(&[(header::IF_NONE_MATCH, signed.as_str())]), + false, + ), + ( + request_headers(&[(header::IF_MATCH, "W/\"sig-00\"")]), + false, + ), + ] { + assert_eq!( + claim(&services, &Method::GET, &headers).names_signed_bytes(), + names_signed, + "{headers:?}" + ); + } +} + +#[test] +fn answers_if_none_match_against_the_signed_tag_only() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[( + header::IF_NONE_MATCH, + format!("\"other\", W/{}", signed_etag()).as_str(), + )]), + whole_origin(), + ); + assert_eq!( + response.status(), + StatusCode::NOT_MODIFIED, + "a weak match of the signed tag is a 304" + ); + assert!(body.is_empty(), "a 304 carries no body"); + assert_eq!(header(&response, "content-length"), None); + assert_eq!(header(&response, "etag"), Some(signed_etag().as_str())); + assert_eq!( + header(&response, "x-test-manifest"), + Some("instance-1"), + "a 304 carries the signer's headers" + ); + + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::IF_NONE_MATCH, ORIGIN_ETAG)]), + whole_origin(), + ); + assert_eq!( + response.status(), + StatusCode::OK, + "the origin's tag never matches signed bytes" + ); + assert_eq!(body, signed_bytes()); +} + +#[test] +fn fails_if_match_that_names_another_representation() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::IF_MATCH, ORIGIN_ETAG)]), + whole_origin(), + ); + assert_eq!(response.status(), StatusCode::PRECONDITION_FAILED); + assert!(body.is_empty()); + + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, _) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::IF_MATCH, signed_etag().as_str())]), + whole_origin(), + ); + assert_eq!( + response.status(), + StatusCode::OK, + "the signed tag satisfies If-Match" + ); +} + +#[test] +fn signs_the_whole_behind_a_part_that_can_be_kept_and_ranges_the_signed_bytes() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + let headers = image_request(&[(header::RANGE, "bytes=0-3")]); + let claim = claim(&services, &Method::GET, &headers); + + let Delivery::FetchWhole(fallback) = delivery(&claim, &services, part_origin()) else { + panic!("should ask for the whole representation of a part it can keep"); + }; + assert!(signer.jobs().is_empty(), "a part is never signed"); + + let delivered = futures::executor::block_on(claim.deliver_whole( + &services, + &identity(), + whole_origin(), + fallback, + )); + let (response, body) = split(delivered); + assert_eq!( + signer.jobs()[0].source, + SOURCE, + "should sign the whole answer's exact bytes" + ); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + body, + signed_bytes()[0..=3], + "should range the signed bytes, not the origin's" + ); + assert_eq!(header(&response, "etag"), Some(signed_etag().as_str())); + assert_eq!( + header(&response, "accept-ranges"), + Some("bytes"), + "a kept representation offers ranges" + ); +} + +#[test] +fn passes_a_part_through_where_nothing_would_keep_the_signed_whole() { + let cases: [(RuntimeServices, Response, &str); 3] = [ + ( + StubSigner::new(Behavior::Sign, 1024).installed_in(noop_services()), + part_origin(), + "no cache keeps values", + ), + ( + keeping(&StubSigner::new(Behavior::Sign, 1024)), + origin_response( + StatusCode::PARTIAL_CONTENT, + EdgeBody::from(SOURCE[0..=3].to_vec()), + &[ + (header::CONTENT_RANGE, "bytes 0-3/18"), + (header::CACHE_CONTROL, "no-cache"), + ], + ), + "the origin gave no shared freshness", + ), + ( + keeping(&StubSigner::new(Behavior::Sign, 1024)), + origin_response( + StatusCode::PARTIAL_CONTENT, + EdgeBody::from(b"part".to_vec()), + &[(header::CONTENT_RANGE, "bytes 0-3/50000000")], + ), + "the whole is over the limit", + ), + ]; + for (services, part, case) in cases { + let claim = claim( + &services, + &Method::GET, + &image_request(&[(header::RANGE, "bytes=0-3")]), + ); + let Delivery::Send(response) = delivery(&claim, &services, part) else { + panic!("{case}: should not read the whole origin for a part"); + }; + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT, "{case}"); + assert_eq!( + header(&response, "etag"), + Some(ORIGIN_ETAG), + "{case}: keeps the origin's part" + ); + } +} + +#[test] +fn asks_for_the_signed_tail_the_origin_refused_as_past_its_own_length() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let tail = signed_bytes().len() - 2; + let range = format!("bytes={tail}-"); + let headers = request_headers(&[(header::RANGE, range.as_str())]); + let claim = claim(&services, &Method::GET, &headers); + let refusal = origin_response( + StatusCode::RANGE_NOT_SATISFIABLE, + EdgeBody::empty(), + &[(header::CONTENT_RANGE, "bytes */18")], + ); + + let Delivery::FetchWhole(fallback) = delivery(&claim, &services, refusal) else { + panic!("a range past the origin's length may name the signed tail"); + }; + let delivered = futures::executor::block_on(claim.deliver_whole( + &services, + &identity(), + whole_origin(), + fallback, + )); + let (response, body) = split(delivered); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(body, signed_bytes()[tail..], "should send the signed tail"); +} + +#[test] +fn sends_the_origins_part_without_validators_when_the_whole_fetch_cannot_be_signed() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + let claim = claim( + &services, + &Method::GET, + &image_request(&[(header::RANGE, "bytes=0-3")]), + ); + + let mut missing = whole_origin(); + *missing.status_mut() = StatusCode::NOT_FOUND; + let delivered = futures::executor::block_on(claim.deliver_whole( + &services, + &identity(), + missing, + part_origin(), + )); + let (response, body) = split(delivered); + assert_eq!( + response.status(), + StatusCode::PARTIAL_CONTENT, + "should keep the origin's part" + ); + assert_eq!(body, SOURCE[0..=3]); + assert_eq!( + header(&response, "etag"), + None, + "an unsigned fallback carries no validator to pin it" + ); + assert_eq!( + header(&response, "cache-control"), + Some("private, max-age=60") + ); + assert!(signer.jobs().is_empty()); +} + +#[test] +fn answers_head_from_the_signed_whole_with_no_body() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + let claim = claim(&services, &Method::HEAD, &image_request(&[])); + let head = origin_response( + StatusCode::OK, + EdgeBody::empty(), + &[(header::CONTENT_LENGTH, "18")], + ); + + let Delivery::FetchWhole(fallback) = delivery(&claim, &services, head) else { + panic!("a HEAD of a bounded type that can be kept should ask for the whole body"); + }; + let delivered = futures::executor::block_on(claim.deliver_whole( + &services, + &identity(), + whole_origin(), + fallback, + )); + let (response, body) = split(delivered); + assert_eq!(response.status(), StatusCode::OK); + assert!(body.is_empty(), "a HEAD carries no body"); + assert_eq!( + header(&response, "content-length"), + Some(signed_bytes().len().to_string().as_str()), + "a HEAD advertises the signed length" + ); + assert_eq!(header(&response, "etag"), Some(signed_etag().as_str())); +} + +#[test] +fn never_signs_a_response_the_origin_marked_as_one_readers() { + let private_markers: [(HeaderName, &str); 4] = [ + (header::SET_COOKIE, "session=1"), + (header::CACHE_CONTROL, "private, max-age=60"), + (header::VARY, "Accept, Cookie"), + (header::VARY, "*"), + ]; + for (name, value) in private_markers { + let signer = StubSigner::new(Behavior::Sign, 1024); + let mut response = whole_origin(); + response + .headers_mut() + .insert(name.clone(), HeaderValue::from_static(value)); + let (response, body) = deliver(&signer, &Method::GET, &HeaderMap::new(), response); + assert!( + signer.jobs().is_empty(), + "{name}: {value} should not reach the signer" + ); + assert_eq!( + body, SOURCE, + "{name}: {value} should go out as the origin sent it" + ); + assert_eq!(header(&response, "etag"), Some(ORIGIN_ETAG)); + } +} + +#[test] +fn reads_the_origins_own_headers_not_the_policy_core_wrote_over_them() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let claim = claim(&services, &Method::GET, &HeaderMap::new()); + let raw = origin_response( + StatusCode::OK, + EdgeBody::empty(), + &[(header::CACHE_CONTROL, "public, max-age=60, no-transform")], + ); + let facts = OriginFacts::of(raw.headers()); + let normalized = origin_response( + StatusCode::OK, + EdgeBody::from(SOURCE.to_vec()), + &[(header::CACHE_CONTROL, "public, max-age=31536000, immutable")], + ); + + futures::executor::block_on(claim.deliver( + &services, + &identity(), + Answered::Origin(facts), + normalized, + )); + assert!( + signer.jobs().is_empty(), + "the origin's no-transform stands under core's policy" + ); +} + +#[test] +fn claims_nothing_without_a_signer_with_authorization_or_off_its_paths() { + let settings = create_test_settings(); + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let none = |services: &RuntimeServices, method: &Method, path: &str, headers: &HeaderMap| { + MediaClaim::of_request(&settings, services, method, path, headers, true).is_none() + }; + assert!( + none(&noop_services(), &Method::GET, PATH, &HeaderMap::new()), + "services without a signer claim nothing" + ); + assert!( + none( + &services, + &Method::GET, + PATH, + &request_headers(&[(header::AUTHORIZATION, "Bearer reader-token")]) + ), + "a request bound to one reader's credentials is never claimed" + ); + assert!( + none(&services, &Method::GET, "/api/items", &HeaderMap::new()), + "a path the signer leaves alone" + ); + assert!( + none(&services, &Method::POST, PATH, &HeaderMap::new()), + "only GET and HEAD are claimed" + ); +} + +#[test] +fn tells_a_media_request_by_destination_accept_or_extension() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let settings = create_test_settings(); + let looks = |path: &str, headers: &HeaderMap| { + MediaClaim::of_request(&settings, &services, &Method::GET, path, headers, true) + .expect("should claim") + .looks_like_media() + }; + assert!( + looks(PATH, &image_request(&[])), + "an image destination is media" + ); + assert!(!looks( + "/photo.jpg", + &request_headers(&[("sec-fetch-dest".parse().expect("should parse"), "script")]) + )); + assert!(looks( + PATH, + &request_headers(&[(header::ACCEPT, "image/avif,image/webp,*/*")]) + )); + assert!( + looks("/clips/intro.MP4", &HeaderMap::new()), + "a media extension is media" + ); + assert!( + !looks(PATH, &request_headers(&[(header::ACCEPT, "text/html,*/*")])), + "a page is not" + ); +} + +#[test] +fn leaves_untransformable_encoded_and_unhandled_bodies_as_the_origin_sent_them() { + let extras: [(HeaderName, &str); 3] = [ + (header::CACHE_CONTROL, "public, no-transform"), + (header::CONTENT_ENCODING, "gzip"), + (header::CONTENT_TYPE, "text/plain"), + ]; + for (name, value) in extras { + let signer = StubSigner::new(Behavior::Sign, 1024); + let mut response = whole_origin(); + response + .headers_mut() + .insert(name.clone(), HeaderValue::from_static(value)); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, "bytes=0-1")]), + response, + ); + assert!( + signer.jobs().is_empty(), + "{name}: {value} should not reach the signer" + ); + assert_eq!( + response.status(), + StatusCode::OK, + "{name}: {value} keeps the origin's status" + ); + assert_eq!(body, SOURCE); + } +} + +#[test] +fn passes_other_statuses_through_untouched() { + for status in [ + StatusCode::NOT_FOUND, + StatusCode::NOT_MODIFIED, + StatusCode::RANGE_NOT_SATISFIABLE, + ] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let response = origin_response(status, EdgeBody::empty(), &[]); + let (response, _) = deliver(&signer, &Method::GET, &HeaderMap::new(), response); + assert_eq!( + response.status(), + status, + "a reader that asked no range gets the origin's {status}" + ); + assert_eq!(header(&response, "etag"), Some(ORIGIN_ETAG)); + assert!(signer.jobs().is_empty()); + } +} + +#[test] +fn sends_the_origin_bytes_unsigned_without_validators_when_signing_fails() { + for behavior in [Behavior::Fail, Behavior::ReservedHeader] { + let signer = StubSigner::new(behavior, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, "bytes=0-5")]), + whole_origin(), + ); + assert_eq!( + signer.jobs().len(), + 1, + "{behavior:?} should have asked the signer" + ); + assert_eq!( + response.status(), + StatusCode::OK, + "{behavior:?} keeps the origin's own answer" + ); + assert_eq!(body, SOURCE); + assert_eq!( + header(&response, "etag"), + None, + "{behavior:?} drops the origin's validator" + ); + assert_eq!( + header(&response, "cache-control"), + Some("private, max-age=60") + ); + assert_eq!(header(&response, "x-test-manifest"), None); + assert_eq!(header(&response, "repr-digest"), None); + } +} + +#[test] +fn captures_a_streamed_body_within_the_limit() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let chunks = SOURCE + .chunks(5) + .map(Bytes::copy_from_slice) + .collect::>(); + let (response, body) = deliver( + &signer, + &Method::GET, + &HeaderMap::new(), + origin_response( + StatusCode::OK, + EdgeBody::stream(futures::stream::iter(chunks)), + &[], + ), + ); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + signer.jobs()[0].source, + SOURCE, + "should join the streamed chunks exactly" + ); + assert_eq!(body, signed_bytes()); +} + +#[test] +fn streams_a_body_over_the_limit_unsigned_and_whole() { + let signer = StubSigner::new(Behavior::Sign, 8); + let chunks = SOURCE + .chunks(4) + .map(Bytes::copy_from_slice) + .collect::>(); + let (response, body) = deliver( + &signer, + &Method::GET, + &HeaderMap::new(), + origin_response( + StatusCode::OK, + EdgeBody::stream(futures::stream::iter(chunks)), + &[], + ), + ); + assert!( + signer.jobs().is_empty(), + "an oversized body should not reach the signer" + ); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + body, SOURCE, + "should send every byte read and unread, in order" + ); + assert_eq!(header(&response, "etag"), Some(ORIGIN_ETAG)); +} + +#[test] +fn keeps_a_failed_stream_failing_after_the_bytes_read() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let claim = claim(&services, &Method::GET, &HeaderMap::new()); + let stream = futures::stream::iter(vec![ + Ok(Bytes::from_static(b"first")), + Err(std::io::Error::other("origin reset")), + ]); + let Delivery::Send(delivered) = delivery( + &claim, + &services, + origin_response(StatusCode::OK, EdgeBody::from_stream(stream), &[]), + ) else { + panic!("should send the origin's answer"); + }; + let EdgeBody::Stream(mut body) = delivered.into_body() else { + panic!("should keep a failed body streaming"); + }; + futures::executor::block_on(async { + let first = body.next().await.expect("should yield the bytes read"); + assert_eq!( + first.expect("should read the first chunk"), + Bytes::from_static(b"first") + ); + assert!( + body.next() + .await + .expect("should yield the failure") + .is_err(), + "should hand the reader the origin's failure, never a short body as whole" + ); + }); + assert!( + signer.jobs().is_empty(), + "a body that failed is never signed" + ); +} + +/// Signs once through `services` for a reader whose request is `headers`, +/// then gives back what a later image request finds before the origin. +fn sign_then_look_up( + signer: &Arc, + services: &RuntimeServices, + headers: &HeaderMap, + shared: bool, + origin: Response, +) -> Option> { + let first = claim_shared(services, &Method::GET, headers, shared); + let _ = delivery(&first, services, origin); + assert_eq!(signer.jobs().len(), 1, "should sign the first request"); + let later = claim(services, &Method::GET, &image_request(&[])); + futures::executor::block_on(later.kept(services, &identity())) +} + +#[test] +fn answers_a_later_range_from_the_kept_signed_representation_before_the_origin() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + + let kept = sign_then_look_up( + &signer, + &services, + &image_request(&[]), + true, + whole_origin(), + ) + .expect("should keep a fresh signed representation"); + let age = header(&kept, "age") + .and_then(|age| age.parse::().ok()) + .expect("a kept copy carries its age"); + assert!( + age <= 1, + "a copy kept just now has almost no age, got {age}" + ); + + let ranged = image_request(&[(header::RANGE, "bytes=1-4")]); + let claim = claim(&services, &Method::GET, &ranged); + let Delivery::Send(response) = + futures::executor::block_on(claim.deliver(&services, &identity(), Answered::Kept, kept)) + else { + panic!("a kept representation is answered at once"); + }; + let (response, body) = split(response); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(body, signed_bytes()[1..=4]); + assert_eq!(header(&response, "etag"), Some(signed_etag().as_str())); + assert_eq!(header(&response, "x-test-manifest"), Some("instance-1")); + assert_eq!(signer.jobs().len(), 1, "the kept copy is not signed again"); +} + +#[test] +fn keeps_and_serves_nothing_for_a_request_whose_answer_could_be_one_readers() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + assert!( + sign_then_look_up( + &signer, + &services, + &image_request(&[]), + false, + whole_origin() + ) + .is_none(), + "an answer to a cookie-bearing request is never kept for other readers" + ); + + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + assert!( + sign_then_look_up( + &signer, + &services, + &image_request(&[]), + true, + whole_origin() + ) + .is_some() + ); + let cookie_bearing = claim_shared(&services, &Method::GET, &image_request(&[]), false); + assert!( + futures::executor::block_on(cookie_bearing.kept(&services, &identity())).is_none(), + "a cookie-bearing request is never answered from a copy kept for others" + ); + let revalidating = claim( + &services, + &Method::GET, + &image_request(&[(header::CACHE_CONTROL, "no-cache")]), + ); + assert!( + futures::executor::block_on(revalidating.kept(&services, &identity())).is_none(), + "a request asking for revalidation is not answered from a kept copy" + ); + let not_media = claim(&services, &Method::GET, &HeaderMap::new()); + assert!( + futures::executor::block_on(not_media.kept(&services, &identity())).is_none(), + "a request that does not look like media is not looked up" + ); +} + +#[test] +fn serves_nothing_kept_once_the_signers_namespace_changes_or_fails() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + assert!( + sign_then_look_up( + &signer, + &services, + &image_request(&[]), + true, + whole_origin() + ) + .is_some() + ); + let claim = claim(&services, &Method::GET, &image_request(&[])); + + signer.set_namespace(Ok("policy-2".to_owned())); + assert!( + futures::executor::block_on(claim.kept(&services, &identity())).is_none(), + "a changed policy finds nothing kept under the old one" + ); + + signer.set_namespace(Err(MediaSigningFailure { reason: "revoked" })); + assert!( + futures::executor::block_on(claim.kept(&services, &identity())).is_none(), + "a revoked signer serves nothing kept" + ); +} + +#[test] +fn keeps_nothing_the_origin_did_not_make_shareably_fresh() { + for cache_control in ["no-cache", "public", "public, max-age=0"] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + let mut origin = whole_origin(); + origin.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static(cache_control), + ); + assert!( + sign_then_look_up(&signer, &services, &image_request(&[]), true, origin).is_none(), + "`{cache_control}` should keep no signed copy" + ); + } + + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + let mut origin = whole_origin(); + origin + .headers_mut() + .insert(header::VARY, HeaderValue::from_static("Accept")); + assert!( + sign_then_look_up(&signer, &services, &image_request(&[]), true, origin).is_none(), + "an answer that varies on the reader's Accept keeps no single signed copy" + ); +} + +#[test] +fn a_kept_copy_ages_and_is_refused_when_its_body_is_not_the_one_signed() { + let mut headers = HeaderMap::new(); + let digest = Sha256::digest(signed_bytes()); + headers.insert( + header::ETAG, + HeaderValue::from_str(&signed_entity_tag(digest.as_slice())).expect("should build tag"), + ); + let body = Bytes::from(signed_bytes()); + let encoded = encode_kept(&KeptRecord { + kept_at: 1_000, + age: 5, + headers: &headers, + body: &body, + }); + + let kept = decode_kept(&encoded, 1_030).expect("should read an intact copy"); + assert_eq!( + header(&kept, "age"), + Some("35"), + "should add the time kept to the origin's age" + ); + + let mut altered = encoded.to_vec(); + let last = altered.len() - 1; + altered[last] ^= 0x01; + assert!( + decode_kept(&Bytes::from(altered), 1_030).is_none(), + "a body that no longer hashes to its tag is never served" + ); +} + +#[test] +fn strips_every_header_that_shortens_the_origin_answer() { + let mut headers = request_headers(&[ + (header::RANGE, "bytes=0-1"), + (header::IF_RANGE, ORIGIN_ETAG), + (header::IF_NONE_MATCH, ORIGIN_ETAG), + (header::IF_MODIFIED_SINCE, "Tue, 01 Sep 2026 00:00:00 GMT"), + (header::IF_MATCH, ORIGIN_ETAG), + (header::IF_UNMODIFIED_SINCE, "Tue, 01 Sep 2026 00:00:00 GMT"), + (header::ACCEPT, "image/avif"), + ]); + strip_representation_preconditions(&mut headers); + assert_eq!(headers.len(), 1, "should keep only the unrelated header"); + assert!(headers.contains_key(header::ACCEPT)); +} + +/// A manifest the origin already links its bytes to. +const ORIGIN_MANIFEST_LINK: &str = "; rel=\"c2pa-manifest\""; + +/// `response` with the origin's own manifest `Link`. +fn linked(mut response: Response) -> Response { + response + .headers_mut() + .append(header::LINK, HeaderValue::from_static(ORIGIN_MANIFEST_LINK)); + response +} + +fn links(response: &Response) -> Vec { + response + .headers() + .get_all(header::LINK) + .iter() + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + .collect() +} + +#[test] +fn keeps_an_origin_manifest_link_and_its_bytes_for_external_and_embeddable_signers() { + // `SignWithLink` stands for an external-only signer, `Sign` for one + // that embeds a manifest and changes the bytes. + for behavior in [Behavior::SignWithLink, Behavior::Sign] { + let signer = StubSigner::new(behavior, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &image_request(&[]), + linked(whole_origin()), + ); + assert!( + signer.jobs().is_empty(), + "{behavior:?}: should never sign bytes that name a manifest of their own" + ); + assert_eq!(body, SOURCE, "{behavior:?}: should send the origin's bytes"); + assert_eq!( + header(&response, "etag"), + Some(ORIGIN_ETAG), + "{behavior:?}: should keep the origin's validator" + ); + assert_eq!( + links(&response), + vec![ORIGIN_MANIFEST_LINK.to_owned()], + "{behavior:?}: should keep only the origin's manifest link" + ); + assert_eq!(header(&response, "x-test-manifest"), None); + assert_eq!(header(&response, "repr-digest"), None); + } +} + +#[test] +fn never_signs_a_whole_fetch_where_an_answer_names_an_origin_manifest() { + let signer = StubSigner::new(Behavior::SignWithLink, 1024); + let services = keeping(&signer); + let first_claim = claim( + &services, + &Method::GET, + &image_request(&[(header::RANGE, "bytes=0-3")]), + ); + let Delivery::Send(response) = delivery(&first_claim, &services, linked(part_origin())) else { + panic!("should never ask for the whole behind a part that names an origin manifest"); + }; + let (response, body) = split(response); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(body, SOURCE[0..=3]); + assert_eq!(links(&response), vec![ORIGIN_MANIFEST_LINK.to_owned()]); + assert!(signer.jobs().is_empty()); + + let tail = signed_bytes().len() - 2; + let range = format!("bytes={tail}-"); + for (refusal_linked, whole_linked) in [(false, true), (true, false)] { + let signer = StubSigner::new(Behavior::SignWithLink, 1024); + let services = signer.installed_in(noop_services()); + let claim = claim( + &services, + &Method::GET, + &request_headers(&[(header::RANGE, range.as_str())]), + ); + let refusal = origin_response( + StatusCode::RANGE_NOT_SATISFIABLE, + EdgeBody::empty(), + &[(header::CONTENT_RANGE, "bytes */18")], + ); + let refusal = if refusal_linked { + linked(refusal) + } else { + refusal + }; + let Delivery::FetchWhole(fallback) = delivery(&claim, &services, refusal) else { + panic!("a range past the origin's length may name the signed tail"); + }; + let whole = if whole_linked { + linked(whole_origin()) + } else { + whole_origin() + }; + let delivered = futures::executor::block_on(claim.deliver_whole( + &services, + &identity(), + whole, + fallback, + )); + let (response, body) = split(delivered); + let case = format!("refusal_linked={refusal_linked} whole_linked={whole_linked}"); + assert!(signer.jobs().is_empty(), "{case}: should sign nothing"); + assert_eq!( + response.status(), + StatusCode::RANGE_NOT_SATISFIABLE, + "{case}: should send the origin's own answer" + ); + assert!(body.is_empty(), "{case}"); + assert_eq!(header(&response, "etag"), Some(ORIGIN_ETAG), "{case}"); + let expected = if refusal_linked { + vec![ORIGIN_MANIFEST_LINK.to_owned()] + } else { + Vec::new() + }; + assert_eq!( + links(&response), + expected, + "{case}: should add no manifest link" + ); + } +} + +/// One ISO BMFF box of `kind` around `body`, with a 32-bit size. +fn bmff_box(kind: &[u8; 4], body: &[u8]) -> Vec { + let size = u32::try_from(body.len() + 8).expect("should fit a 32-bit size"); + let mut bytes = size.to_be_bytes().to_vec(); + bytes.extend_from_slice(kind); + bytes.extend_from_slice(body); + bytes +} + +fn concat(parts: &[Vec]) -> Vec { + parts.concat() +} + +/// A C2PA verifiable segment info `emsg`, as a packager writes it. +fn c2pa_emsg() -> Vec { + bmff_box(b"emsg", b"\0\0\0\0urn:c2pa:verifiable-segment-info\0fseg\0") +} + +#[test] +fn tells_the_parts_of_a_fragmented_stream_from_a_progressive_file() { + let init = concat(&[ + bmff_box(b"ftyp", b"cmfc\0\0\0\0iso6cmfc"), + bmff_box( + b"moov", + &concat(&[ + bmff_box(b"mvhd", &[0; 100]), + bmff_box(b"trak", &bmff_box(b"tkhd", &[0; 84])), + bmff_box(b"mvex", &bmff_box(b"trex", &[0; 24])), + bmff_box(b"pssh", &[0; 32]), + ]), + ), + ]); + let fragment = || { + vec![ + bmff_box(b"sidx", &[0; 32]), + bmff_box(b"moof", &bmff_box(b"mfhd", &[0; 8])), + bmff_box(b"mdat", b"coded-samples"), + ] + }; + let segment = concat(&[vec![bmff_box(b"styp", b"msdh\0\0\0\0msdhmsix")], fragment()].concat()); + let segment_with_emsg = concat( + &[ + vec![bmff_box(b"styp", b"msdh\0\0\0\0msdhmsix"), c2pa_emsg()], + fragment(), + ] + .concat(), + ); + let chunk_without_styp = concat(&fragment()); + let progressive = concat(&[ + bmff_box(b"ftyp", b"isom\0\0\0\0isomavc1"), + bmff_box(b"moov", &bmff_box(b"trak", &[0; 16])), + bmff_box(b"mdat", b"coded-samples"), + ]); + let fragmented_file = concat(&[ + bmff_box(b"ftyp", b"iso6\0\0\0\0iso6dash"), + bmff_box(b"moov", &bmff_box(b"mvex", &[0; 8])), + bmff_box(b"moof", &[0; 8]), + bmff_box(b"mdat", b"coded-samples"), + ]); + let mut large_size = init.clone(); + // The same init with its `ftyp` given a 64-bit size. + let ftyp_body = b"cmfc\0\0\0\0iso6cmfc"; + let mut large_ftyp = 1_u32.to_be_bytes().to_vec(); + large_ftyp.extend_from_slice(b"ftyp"); + large_ftyp.extend_from_slice(&(16 + ftyp_body.len() as u64).to_be_bytes()); + large_ftyp.extend_from_slice(ftyp_body); + large_size.splice(0..8 + ftyp_body.len(), large_ftyp); + let mut truncated = segment.clone(); + truncated.truncate(segment.len() - 3); + let mut zero_sized_last = segment.clone(); + let mdat_at = zero_sized_last.len() - (8 + b"coded-samples".len()); + zero_sized_last[mdat_at..mdat_at + 4].copy_from_slice(&0_u32.to_be_bytes()); + let too_many = bmff_box(b"free", &[]).repeat(MAX_BMFF_BOXES + 1); + + for (case, bytes, expected) in [ + ("Shaka-style init", &init, true), + ("init with a 64-bit ftyp size", &large_size, true), + ("media segment", &segment, true), + ("media segment with a C2PA emsg", &segment_with_emsg, true), + ("CMAF chunk without styp", &chunk_without_styp, true), + ("segment whose mdat runs to the end", &zero_sized_last, true), + ("progressive file", &progressive, false), + ("fragmented file with its media", &fragmented_file, false), + ("truncated segment", &truncated, false), + ("too many boxes", &too_many, false), + ("an image", &SOURCE.to_vec(), false), + ("nothing", &Vec::new(), false), + ] { + assert_eq!( + is_fragmented_stream_part(bytes), + expected, + "{case}: should be told correctly" + ); + } +} + +#[test] +fn origin_freshness_keeps_browser_and_shared_age_and_processing_independent() { + let receipt_seconds = 1_700_000_000; + let receipt = UNIX_EPOCH + Duration::from_secs(receipt_seconds); + for (control, age, date_age, elapsed, browser, shared) in [ + ( + "public, max-age=60, s-maxage=3600", + Some("0"), + None, + 0, + 60, + 3600, + ), + ( + "public, max-age=60, s-maxage=3600", + Some("5"), + Some(120), + 0, + 0, + 3480, + ), + ( + "public, max-age=60, s-maxage=3600", + None, + Some(120), + 0, + 0, + 3480, + ), + ( + "public, max-age=60, s-maxage=3600", + Some("200"), + Some(120), + 0, + 0, + 3400, + ), + ( + "public, max-age=90, s-maxage=180", + Some("10"), + None, + 40, + 40, + 130, + ), + ( + "public, max-age=90, s-maxage=180", + Some("10"), + None, + 90, + 0, + 80, + ), + ( + "public, max-age=90, s-maxage=10", + Some("15"), + None, + 0, + 75, + 0, + ), + ("public, max-age=90", Some("70"), None, 0, 20, 20), + ("public, max-age=90", Some("100"), None, 0, 0, 0), + ("public, s-maxage=90", None, None, 0, 0, 90), + ("public, max-age=bad, s-maxage=90", None, None, 0, 0, 90), + ("public, max-age=60, s-maxage=bad", None, None, 0, 60, 0), + ("public, max-age=60, s-maxage", None, None, 0, 60, 0), + ("public, max-age=60, max-age=90", None, None, 0, 0, 0), + ("public, max-age=60", Some("bad"), None, 0, 0, 0), + ("no-cache, max-age=90", Some("0"), None, 0, 0, 0), + ("public", Some("0"), None, 0, 0, 0), + ] { + let mut headers = HeaderMap::new(); + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_str(control).expect("control"), + ); + if let Some(age) = age { + headers.insert(header::AGE, HeaderValue::from_str(age).expect("age")); + } + if let Some(date_age) = date_age { + let date = httpdate::fmt_http_date( + std::time::UNIX_EPOCH + Duration::from_secs(receipt_seconds - date_age), + ); + headers.insert(header::DATE, HeaderValue::from_str(&date).expect("date")); + } + assert_eq!( + OriginFacts::of_at(&headers, receipt).remaining_freshness(Duration::from_secs(elapsed)), + OriginFreshness { + browser: Duration::from_secs(browser), + shared: Duration::from_secs(shared), + }, + "{control}, Age {age:?}, apparent age {date_age:?}, processing {elapsed}", + ); + } +} + +#[test] +fn invalid_origin_date_cannot_gain_freshness_when_rebased() { + let mut headers = HeaderMap::new(); + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=3600"), + ); + headers.insert(header::DATE, HeaderValue::from_static("not a date")); + assert_eq!( + OriginFacts::of_at(&headers, UNIX_EPOCH).remaining_freshness(Duration::ZERO), + OriginFreshness::default(), + ); +} diff --git a/crates/trusted-server-core/src/middleware.rs b/crates/trusted-server-core/src/middleware.rs index bf7717dc6a..a0a072eb4d 100644 --- a/crates/trusted-server-core/src/middleware.rs +++ b/crates/trusted-server-core/src/middleware.rs @@ -46,7 +46,7 @@ use std::rc::Rc; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Duration; -use http::{HeaderName, HeaderValue, StatusCode}; +use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; use crate::attestation::PlatformIdentity; @@ -59,6 +59,17 @@ use crate::streaming_processor::StreamProcessor; /// unless it says it handles another. pub const HTML_MEDIA_TYPE: &str = "text/html"; +/// A module's exact original byte ranges that no document transform may edit. +/// Called on the complete, bounded document before core or middleware runs. +/// Each classifier returns ordered, disjoint, nonempty ranges inside the input. +/// An error refuses the document. Core treats ranges as opaque; classifications +/// from different middleware are combined, including overlapping ranges. +/// +/// Transforms see an opaque comment in place of each protected range. A +/// middleware that signs or hashes raw whole HTML must not bind that intermediate +/// view and assume its binding remains valid after original bytes are restored. +pub type SourceProtection = fn(&[u8]) -> Result>, std::io::Error>; + /// Whether a middleware may run on a response of `media_type`, written as a /// bare `type/subtype` in lower case. /// @@ -144,6 +155,23 @@ pub struct MiddlewareContext<'a> { /// sends, the same for a page served from the store as for one fetched /// for this request. pub origin_host: &'a str, + /// The status of the response the document is the body of. A page from + /// the store of shared templates was stored from a `200`. + pub status: StatusCode, + /// Whether the origin marked its response as one reader's, by setting a + /// cookie or by a `Cache-Control` that forbids shared storage. Read from + /// the origin's own headers, before core writes its cache policy on the + /// response, so a middleware that sends the document elsewhere can leave + /// a reader's own page alone. A page from the store of shared templates + /// was never one reader's. + pub origin_private: bool, + /// The headers of the reader's response as core will send them, being + /// the origin's after core moved its addresses to the reader's host, for + /// a middleware whose change has to respect a header such as + /// `Content-Security-Policy`. `None` where core cannot say, being the + /// fetch phase of a page that may be stored and a page replayed from the + /// store, which a middleware treats as headers it does not know. + pub response_headers: Option<&'a HeaderMap>, /// What the adapter reports about the platform the deployment runs on, /// as the attestation evidence carries it. pub platform: &'a PlatformIdentity, @@ -557,6 +585,13 @@ pub trait Middleware: Send + Sync { false } + /// Classifies source bytes that must survive all document transformations + /// unchanged. Default off. Only middleware in the applicable chains may + /// request this hold, including serve middleware before a template is built. + fn source_protection(&self) -> Option { + None + } + /// Whether this middleware moves the origin's address to the reader's /// host itself, in a body of a media type it handles other than HTML. /// Core then leaves the addresses in such a body alone, whether or not @@ -917,6 +952,15 @@ impl MiddlewareChain { self.middleware.is_empty() } + /// The original-source classifiers selected by this document's chain. + #[must_use] + pub fn source_protections(&self) -> Vec { + self.middleware + .iter() + .filter_map(|middleware| middleware.source_protection()) + .collect() + } + /// Whether a middleware in the chain writes the whole document, see /// [`Middleware::writes_whole_document`]. #[must_use] @@ -1103,6 +1147,9 @@ pub mod test_support { request_host: REQUEST_HOST, request_scheme: "https", origin_host: ORIGIN_HOST, + status: http::StatusCode::OK, + origin_private: false, + response_headers: None, platform, document_state, max_buffered_script_bytes: 16 * 1024 * 1024, diff --git a/crates/trusted-server-core/src/middleware/tests.rs b/crates/trusted-server-core/src/middleware/tests.rs index 72814b6d26..6275024e64 100644 --- a/crates/trusted-server-core/src/middleware/tests.rs +++ b/crates/trusted-server-core/src/middleware/tests.rs @@ -67,6 +67,9 @@ fn reader(request_state: &IntegrationRequestState) -> ReaderDocument<'_> { request_host: REQUEST_HOST, request_scheme: "https", origin_host: ORIGIN_HOST, + status: StatusCode::OK, + origin_private: false, + response_headers: None, platform: test_support::platform(), request_state, max_buffered_script_bytes: 16 * 1024 * 1024, @@ -2388,3 +2391,119 @@ fn an_html_entry_ahead_does_not_shadow_the_entry_that_makes_the_file() { "should make the file from the text/plain entry, not as HTML" ); } + +/// A format-agnostic source policy supplied by a test module. Core knows only +/// its returned byte range, not why that subtree must remain unchanged. +struct ProtectedFixture; + +const PROTECTED_FIXTURE: &str = "

keep

keep

"; + +impl Middleware for ProtectedFixture { + fn middleware_id(&self) -> &'static str { + "fixture.protected" + } + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + fn create(&self, _: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } + fn source_protection(&self) -> Option { + Some(|original| { + Ok(original + .windows(PROTECTED_FIXTURE.len()) + .position(|bytes| bytes == PROTECTED_FIXTURE.as_bytes()) + .map(|start| start..start + PROTECTED_FIXTURE.len()) + .into_iter() + .collect()) + }) + } +} + +#[test] +fn source_policy_protects_before_core_fetch_and_warm_serve_descendant_handlers() { + use crate::html_processor::create_html_processor_with_source_protection; + + let fetch = chain_of(vec![ + element("fixture.fetch-attribute", "a.edit", "href", |_| { + AttributeRewriteAction::Replace("fetch-edited".to_owned()) + }), + text("fixture.fetch-text", "p.edit", || { + |_, last| { + if last { + ScriptRewriteAction::Replace("fetch-edited".to_owned()) + } else { + ScriptRewriteAction::remove_node() + } + } + }), + ]); + let serve = serve_chain_of(vec![ + Arc::new(ProtectedFixture), + element("fixture.serve-attribute", "a.edit", "href", |_| { + AttributeRewriteAction::Replace("serve-edited".to_owned()) + }), + text("fixture.serve-text", "p.edit", || { + |_, last| { + if last { + ScriptRewriteAction::Replace("serve-edited".to_owned()) + } else { + ScriptRewriteAction::remove_node() + } + } + }), + ]); + let html = format!( + "{PROTECTED_FIXTURE}\ + outside\ +

outside

" + ); + // The source policy lives in serve, but must already cover fetch/template + // construction, without running any of serve's actions early. + let (mut processor, _) = + create_html_processor_with_source_protection(config(), &fetch, None, &[&fetch, &serve]) + .expect("plan template"); + for chunk in html.as_bytes().chunks(3) { + assert!( + processor + .process_chunk(chunk, false) + .expect("hold original") + .is_empty() + ); + } + let template = String::from_utf8(processor.process_chunk(&[], true).expect("finish template")) + .expect("UTF-8"); + assert!(template.contains(PROTECTED_FIXTURE)); + assert!(template.contains("href=\"fetch-edited\"")); + assert!(template.contains(">fetch-edited

")); + assert!(!template.contains("serve-edited")); + assert!(!template.contains("