From e6af5f6a77063ba025bbc10a1b9c07db8aea4466 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 14 Oct 2026 23:51:13 +0100 Subject: [PATCH 1/7] AWS signing is a shared crate and core signs through an origin signer Core signed reads from the publisher's origin and from each asset route's origin with one cloud vendor's signing scheme, and its settings knew that scheme's region and credential key names, so a signed read with another scheme was a change to core. Core now keeps an origin authentication interface. OriginSigner, in the new origin_auth module, signs the headers of a request for a method on a final address and removes every header in the scheme's own namespace. A source's auth table keeps its shape: type names the signer, origin_query is read by core, and the rest of the table belongs to the signer named. A crate offers a signer on its builder with with_origin_signer, which also declares the settings of the signer's table that hold the name of a secret and the key name each takes when left out. The registry builds the signer where a source names it, so a type nothing offers is refused at startup naming the signers on offer, and a table the signer cannot use is refused naming the setting. The declared secrets are looked up as the settings load at both auth paths, a left out setting with a default is written in before the lookup, a written one naming no key fails deploy validation naming its path, and the configuration page masks them. The s3_sigv4 signer moves to crates/shared/aws, the package trusted-server-shared-aws, with the same settings, defaults, secret key references and signing. The stock build offers it through crates/trusted-server-modules, so a deployment's settings read as before. The deprecated spelling s3_sig_v4 is no longer read. Core's tests sign through a fixture signer the origin_auth module keeps for them. The vendor crate's tests cover the published signature example, the registry and deploy refusals, and the lookup and masking of its credentials. Design: docs/superpowers/specs/2026-10-14-origin-signing-design.md --- .cargo/config.toml | 12 +- CHANGELOG.md | 1 + Cargo.lock | 20 + Cargo.toml | 2 + crates/shared/README.md | 18 + crates/shared/aws/Cargo.toml | 39 + crates/shared/aws/src/lib.rs | 690 +++++++++++++ .../s3_sigv4.rs => shared/aws/src/sigv4.rs} | 180 ++-- crates/trusted-server-core/src/config.rs | 186 +--- .../trusted-server-core/src/config_payload.rs | 173 ++-- crates/trusted-server-core/src/host_header.rs | 2 +- .../src/inspect/config/tests.rs | 63 +- .../src/integrations/mod.rs | 22 + .../src/integrations/registry.rs | 47 +- crates/trusted-server-core/src/lib.rs | 2 +- .../trusted-server-core/src/module_secrets.rs | 11 +- crates/trusted-server-core/src/origin_auth.rs | 943 ++++++++++++++++++ crates/trusted-server-core/src/proxy.rs | 322 +++--- crates/trusted-server-core/src/publisher.rs | 104 +- crates/trusted-server-core/src/settings.rs | 429 +++----- crates/trusted-server-core/src/source.rs | 159 ++- .../trusted-server-core/src/test_support.rs | 8 +- crates/trusted-server-modules/Cargo.toml | 1 + crates/trusted-server-modules/src/lib.rs | 4 + docs/guide/api-reference.md | 5 +- docs/guide/asset-routes.md | 8 +- docs/guide/configuration.md | 62 +- docs/guide/integration-guide.md | 43 + .../specs/2026-10-14-origin-signing-design.md | 60 ++ trusted-server.example.toml | 14 +- 30 files changed, 2670 insertions(+), 960 deletions(-) create mode 100644 crates/shared/README.md create mode 100644 crates/shared/aws/Cargo.toml create mode 100644 crates/shared/aws/src/lib.rs rename crates/{trusted-server-core/src/s3_sigv4.rs => shared/aws/src/sigv4.rs} (65%) create mode 100644 crates/trusted-server-core/src/origin_auth.rs create mode 100644 docs/superpowers/specs/2026-10-14-origin-signing-design.md diff --git a/.cargo/config.toml b/.cargo/config.toml index cc11c8e77d..99d4f9f781 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,10 +26,10 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-shared-aws -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-shared-aws -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-shared-aws -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-shared-aws -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-shared-aws -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-shared-aws -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index 6373a277bd..a66855ab6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Breaking for a build that links its own crates:** a source's `auth` table names an origin signer a crate offers. Core keeps the interface, `OriginSigner` in `trusted_server_core::origin_auth`, and signs the publisher's origin requests and each asset route's through it where it signed before. `type` names the signer, `origin_query` is read by core, and the rest of the table is the signer's. The `s3_sigv4` signer moves to `crates/shared/aws`, keeping its settings, defaults and secret-store key references, and a stock build still offers it through `crates/trusted-server-modules`, so a deployment's settings read as before. A build that composes its own modules adds `trusted_server_shared_aws::builder()`, or a `type` of `s3_sigv4` is refused at startup naming the signers on offer. A table the signer cannot use is refused at startup and by `ts config validate` naming the setting. The deprecated spelling `s3_sig_v4` is no longer read. **For crates outside core**, `AssetOriginAuth` and `S3SigV4AuthConfig` are replaced by `OriginAuth`, `SourceLocation::auth` gives an `OriginSigning`, `OriginSettings::location` and `OriginSource::new` take the signers on offer and the setting's name, and a crate offers a signer with `IntegrationBuilder::with_origin_signer`. - **Breaking:** Where content comes from is a source module. `[source]` selects the publisher's, and each `[[proxy.asset_routes]]` entry selects its own in a `source` table of the same shape, with `module` naming the source and its settings in the table beneath. Core supplies `origin`, an HTTP origin, and a module a build links can offer another through `IntegrationRegistrationBuilder::with_source_module`. A source answers for each request where that request's content comes from, and core fetches, caches, rewrites the origin host and follows the redirect rule from that answer. `[source]` is required. The origin settings move into the `origin` table, and a document that still writes one where it was is refused at startup naming the new place. An environment variable that set the address is now `TRUSTED_SERVER__SOURCE__ORIGIN__URL`. The old name is refused where the build reads variables, and `ts config push` ignores it, as it ignores any variable for a key the document does not hold. A path in an asset route's `url` names a folder the target path is added under, where a path was refused, a target path with a `.` or `..` segment under a folder is answered `400`, and a route applies `directory_index` as the publisher does. An existing deployment's stored pages and assets keep their keys. **For crates outside core**, `handle_asset_proxy_request` takes the route's source, which `IntegrationRegistry::route_source` gives, and `Publisher::origin_host`, `origin_host_header` and `origin_query_is_stripped` are replaced by `Settings::origin_source` and the `SourceLocation` a source answers. | Previous | Now | diff --git a/Cargo.lock b/Cargo.lock index 5c87f2354f..5c3457af22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7135,6 +7135,7 @@ dependencies = [ "trusted-server-framework-nextjs", "trusted-server-identity-lockr", "trusted-server-loading-cache-51degrees", + "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", "trusted-server-testing-testlight", @@ -7192,6 +7193,25 @@ dependencies = [ "trusted-server-core", ] +[[package]] +name = "trusted-server-shared-aws" +version = "0.1.0" +dependencies = [ + "chrono", + "edgezero-core", + "error-stack", + "futures", + "hex", + "hmac", + "http", + "log", + "serde", + "serde_json", + "sha2 0.10.9", + "trusted-server-core", + "url", +] + [[package]] name = "trusted-server-tag-google-tag" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b120cf3c28..99074ba451 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/cmp/inmobi", "crates/ads-txt/sellers", "crates/loading-cache/51degrees", + "crates/shared/aws", "crates/tag/google-tag", "crates/tag/google-tag-manager", "crates/testing/testlight", @@ -170,6 +171,7 @@ trusted-server-cmp-didomi = { path = "crates/cmp/didomi" } trusted-server-cmp-inmobi = { path = "crates/cmp/inmobi" } trusted-server-ads-txt-sellers = { path = "crates/ads-txt/sellers" } trusted-server-loading-cache-51degrees = { path = "crates/loading-cache/51degrees" } +trusted-server-shared-aws = { path = "crates/shared/aws" } trusted-server-tag-google-tag = { path = "crates/tag/google-tag" } trusted-server-tag-google-tag-manager = { path = "crates/tag/google-tag-manager" } trusted-server-testing-testlight = { path = "crates/testing/testlight" } diff --git a/crates/shared/README.md b/crates/shared/README.md new file mode 100644 index 0000000000..0e80b7bd64 --- /dev/null +++ b/crates/shared/README.md @@ -0,0 +1,18 @@ +# Shared vendor crates + +What a vendor offers to every crate that wants it lives here, one crate per +vendor at `crates/shared/`, as a vendor's own modules live under the +type folders beside it. A crate here carries no module a section selects. It +offers what it has on its builder, and core calls it through the interface +core declares for that thing. + +- `crates/shared/aws` (`trusted-server-shared-aws`) offers the origin signer + `s3_sigv4`, which a source's `auth` table names to sign a read from S3 with + AWS Signature Version 4. Core's `origin_auth` module declares the signer + interface, the `OriginSigner` trait, and the stock build offers this signer + through `crates/trusted-server-modules`. + +A vendor whose crates share settings, such as an endpoint and a license key, +writes them in one `[shared.]` table, which its builder reads with +`.with_shared_settings(vendor)`. The table's name is the vendor's folder name +here. diff --git a/crates/shared/aws/Cargo.toml b/crates/shared/aws/Cargo.toml new file mode 100644 index 0000000000..6a498e6b6f --- /dev/null +++ b/crates/shared/aws/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "trusted-server-shared-aws" +description = "What Amazon Web Services offers every crate that wants it: the s3_sigv4 origin signer, which signs a read from S3 with AWS Signature Version 4." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +chrono = { workspace = true } +error-stack = { workspace = true } +hex = { workspace = true } +hmac = { workspace = true } +http = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +edgezero-core = { workspace = true, features = ["test-utils"] } +futures = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. Amazon has not yet adopted this crate, so the +# Trusted Server maintainers own it until a vendor owner steps up. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "seeking vendor owner" diff --git a/crates/shared/aws/src/lib.rs b/crates/shared/aws/src/lib.rs new file mode 100644 index 0000000000..514fdf8f9c --- /dev/null +++ b/crates/shared/aws/src/lib.rs @@ -0,0 +1,690 @@ +//! What Amazon Web Services offers every crate that wants it: the `s3_sigv4` +//! origin signer, which signs a read from S3 with AWS Signature Version 4. +//! +//! A source's `auth` table selects it, for the publisher's origin or an asset +//! route's: +//! +//! ```toml +//! [source.origin] +//! url = "https://bucket.storage.example.com/site" +//! +//! [source.origin.auth] +//! type = "s3_sigv4" +//! region = "example-region-1" +//! access_key_id = "origin_access_key_id" +//! secret_access_key = "origin_secret_access_key" +//! # session_token = "origin_session_token" +//! # origin_query = "strip" +//! ``` +//! +//! `region` is the region of the credential scope. `access_key_id`, +//! `secret_access_key` and `session_token` name keys in the default secret +//! store, which core looks up as the settings load, and the first two take +//! their own names when left out. `origin_query` is core's, see +//! `trusted_server_core::origin_auth`. +//! +//! The signer writes header-based authentication with `UNSIGNED-PAYLOAD`, so +//! it signs `GET` and `HEAD` reads and nothing with a body. The address must +//! be the host the service validates the signature for, and `https`, which +//! core requires of every signed source. + +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] + +mod sigv4; + +use std::sync::Arc; +use std::time::SystemTime; + +use error_stack::Report; +use http::{HeaderMap, Method}; +use serde::Deserialize; +use serde_json::{Map, Value}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{IntegrationBuilder, IntegrationRegistration}; +use trusted_server_core::origin_auth::{OriginSigner, OriginSignerModule, SignerSecret}; +use trusted_server_core::redacted::Redacted; +use trusted_server_core::settings::Settings; +use url::Url; + +/// The id of this crate's builder, from the crate's folder. +pub const MODULE: &str = "shared.aws"; + +/// The name an `auth` table writes to select the signer. +pub const SIGNER: &str = "s3_sigv4"; + +/// The settings of the signer's table that name a secret, and the key name +/// each takes when left out. +const SECRETS: &[SignerSecret] = &[ + SignerSecret { + name: "access_key_id", + default: Some("access_key_id"), + }, + SignerSecret { + name: "secret_access_key", + default: Some("secret_access_key"), + }, + SignerSecret { + name: "session_token", + default: None, + }, +]; + +/// The signer, as a builder offers it. +pub static S3_SIGV4: OriginSignerModule = OriginSignerModule { + name: SIGNER, + secrets: SECRETS, + build: build_signer, +}; + +/// The builder a deployment hands to an adapter, which offers the signer +/// whatever the settings say. A stock build lists it in +/// `trusted-server-modules`. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(MODULE, env!("CARGO_PKG_NAME"), no_registration, validate) + .with_origin_signer(&S3_SIGV4) +} + +/// The builder registers no page integration, because no section selects it. +fn no_registration( + _settings: &Settings, +) -> Result, Report> { + Ok(None) +} + +/// Checks every `auth` table naming the signer, for a deployment being +/// validated, so a table the signer cannot use fails `ts config validate` +/// and not the deployment's first start. +fn validate(settings: &Settings) -> Result> { + if let Some(auth) = settings.origin_source().and_then(|origin| origin.auth) + && auth.signer == SIGNER + { + S3SigV4Settings::read(&auth.settings, "source.origin.auth")?; + } + for route in &settings.proxy.asset_routes { + if let Some(auth) = route.origin_source().and_then(|origin| origin.auth) + && auth.signer == SIGNER + { + S3SigV4Settings::read( + &auth.settings, + &format!("{}.origin.auth", route.source_setting()), + )?; + } + } + Ok(false) +} + +fn build_signer( + settings: &Map, + setting: &str, +) -> Result, Report> { + let settings = S3SigV4Settings::read(settings, setting)?; + Ok(Arc::new(S3SigV4Signer { + region: settings.region, + credentials: sigv4::Credentials { + access_key_id: settings.access_key_id.expose().clone(), + secret_access_key: settings.secret_access_key, + session_token: settings.session_token, + }, + })) +} + +fn default_access_key_id() -> Redacted { + Redacted::new("access_key_id".to_string()) +} + +fn default_secret_access_key() -> Redacted { + Redacted::new("secret_access_key".to_string()) +} + +/// The signer's table. The credential fields hold key names in a pushed +/// configuration and the resolved values once the settings have loaded. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct S3SigV4Settings { + /// The region of the credential scope. + pub region: String, + /// Accepted and discarded, because the store comes from the adapter's + /// mapping. + #[serde(default)] + pub secret_store: Option, + /// The access key id. + #[serde(default = "default_access_key_id")] + pub access_key_id: Redacted, + /// The secret access key. + #[serde(default = "default_secret_access_key")] + pub secret_access_key: Redacted, + /// The session token of temporary credentials, when there is one. + #[serde(default)] + pub session_token: Option>, +} + +fn region_is_valid(region: &str) -> bool { + region + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') +} + +impl S3SigV4Settings { + /// Reads and checks the signer's table, where `setting` names it in a + /// refusal. + /// + /// # Errors + /// + /// When the table has a key the signer does not know, a region it cannot + /// sign with, or an empty credential. + pub fn read( + table: &Map, + setting: &str, + ) -> Result> { + let mut settings: Self = + serde_json::from_value(Value::Object(table.clone())).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("[{setting}] could not be read by {SIGNER}: {error}"), + }) + })?; + settings.normalize(); + settings.check(setting)?; + Ok(settings) + } + + fn normalize(&mut self) { + self.region = self.region.trim().to_string(); + if self.secret_store.take().is_some() { + log::warn!( + "{SIGNER} secret_store is deprecated and ignored; credentials resolve through the default secret store" + ); + } + self.access_key_id = Redacted::new(self.access_key_id.expose().trim().to_string()); + self.secret_access_key = Redacted::new(self.secret_access_key.expose().trim().to_string()); + self.session_token = self.session_token.take().and_then(|value| { + let value = value.expose().trim().to_string(); + (!value.is_empty()).then(|| Redacted::new(value)) + }); + } + + fn check(&self, setting: &str) -> Result<(), Report> { + if self.region.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("[{setting}] {SIGNER} region must not be empty"), + })); + } + if !region_is_valid(&self.region) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{setting}] {SIGNER} region must contain only lowercase letters, digits, and '-'" + ), + })); + } + if self.access_key_id.expose().is_empty() || self.secret_access_key.expose().is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("[{setting}] {SIGNER} credentials must not be empty"), + })); + } + Ok(()) + } +} + +/// Signs each request with the settings it was built from. +struct S3SigV4Signer { + region: String, + credentials: sigv4::Credentials, +} + +impl OriginSigner for S3SigV4Signer { + fn sign( + &self, + method: &Method, + url: &Url, + headers: &mut HeaderMap, + now: SystemTime, + ) -> Result<(), Report> { + sigv4::sign_headers(method, url, headers, &self.region, &self.credentials, now) + } + + fn remove_namespace_headers(&self, headers: &mut HeaderMap) { + sigv4::remove_namespace_headers(headers); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use edgezero_core::blob_envelope::BlobEnvelope; + use http::{HeaderMap, HeaderValue, Method, header}; + use serde_json::json; + use trusted_server_core::config::validate_settings_for_deploy_with; + use trusted_server_core::config_payload::settings_from_config_blob_with; + use trusted_server_core::inspect::MASK; + use trusted_server_core::inspect::config::build_view; + use trusted_server_core::integrations::IntegrationRegistry; + use trusted_server_core::origin_auth::OriginAuth; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + use trusted_server_core::platform::{PlatformError, PlatformSecretStore, StoreId, StoreName}; + use trusted_server_core::proxy::handle_asset_proxy_request; + use trusted_server_core::settings::{OriginQueryPolicy, ProxyAssetRoute, Settings}; + use trusted_server_core::source::SourceRequest; + use trusted_server_core::test_support::template::{ + template_with_resolved_required_secrets, uncomment_block, + }; + use trusted_server_core::test_support::tests::{ + create_test_settings, edit_origin, set_route_auth, + }; + + use super::*; + + const STORAGE_URL: &str = "https://bucket.storage.example.com/site"; + const REGION: &str = "test-region-1"; + + /// An `auth` table selecting the signer with resolved credentials, as + /// the settings hold them once loaded. + fn auth(origin_query: Option) -> OriginAuth { + OriginAuth::new(SIGNER) + .with_setting("region", REGION) + .with_setting("access_key_id", "TESTACCESSKEYID") + .with_setting("secret_access_key", "test-secret-access-key") + .with_origin_query(origin_query) + } + + fn settings_signing_the_origin(auth: OriginAuth) -> Settings { + let mut settings = create_test_settings(); + edit_origin(&mut settings, |origin| { + origin.url = STORAGE_URL.to_owned(); + origin.auth = Some(auth); + }); + settings + } + + fn registry(settings: &Settings) -> Result> { + IntegrationRegistry::with_registrations(settings, &[builder()]) + } + + #[test] + fn the_publisher_source_signs_through_the_signer_the_auth_names() { + let settings = settings_signing_the_origin(auth(Some(OriginQueryPolicy::Strip))); + let registry = registry(&settings).expect("should build the registry"); + let source = registry + .publisher_source(&settings) + .expect("should give the publisher's source"); + let headers = HeaderMap::new(); + let location = source + .locate(&SourceRequest { + method: &Method::GET, + path: "/page", + query: "", + headers: &headers, + removed_prefix: None, + }) + .expect("should locate"); + assert!(location.query_is_stripped()); + + let url = Url::parse("https://bucket.storage.example.com/site/page").expect("url"); + let mut sent = HeaderMap::new(); + sent.insert( + header::HOST, + HeaderValue::from_static("bucket.storage.example.com"), + ); + sent.insert("x-amz-meta-reader", HeaderValue::from_static("chosen")); + let signing = location.auth().expect("should sign"); + signing.remove_namespace_headers(&mut sent); + signing + .sign(&Method::GET, &url, &mut sent) + .expect("should sign"); + + let authorization = sent + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .expect("should write an Authorization header"); + assert!( + authorization.starts_with("AWS4-HMAC-SHA256 Credential=TESTACCESSKEYID/") + && authorization.contains(&format!("/{REGION}/s3/aws4_request")), + "should sign with the settings' credentials and region: {authorization}" + ); + assert_eq!( + sent.get("x-amz-content-sha256"), + Some(&HeaderValue::from_static("UNSIGNED-PAYLOAD")) + ); + assert!( + sent.get("x-amz-meta-reader").is_none(), + "a reader's header in the scheme's namespace should be removed" + ); + } + + #[test] + fn an_asset_route_signs_the_address_it_sends() { + futures::executor::block_on(async { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut settings = create_test_settings(); + let mut route = ProxyAssetRoute::new("/assets/", STORAGE_URL); + set_route_auth(&mut route, Some(auth(None))); + settings.proxy.asset_routes.push(route); + let registry = registry(&settings).expect("should build the registry"); + let source = registry + .route_source(&settings, 0) + .expect("should give the route's source"); + let request = http::Request::builder() + .method(Method::GET) + .uri("https://www.example.com/assets/app.js?v=2") + .body(edgezero_core::body::Body::empty()) + .expect("should build the request"); + + handle_asset_proxy_request( + &settings, + &services, + request, + &settings.proxy.asset_routes[0], + source.as_ref(), + ) + .await + .expect("should proxy the signed asset request"); + + assert_eq!( + stub.recorded_request_uris(), + vec!["https://bucket.storage.example.com/site/assets/app.js?v=2"], + "the query is kept unless origin_query strips it" + ); + let headers = stub.recorded_request_headers(); + let sent = &headers[0]; + let value = |name: &str| { + sent.iter() + .find(|(sent_name, _)| sent_name == name) + .map(|(_, value)| value.as_str()) + }; + assert_eq!(value("host"), Some("bucket.storage.example.com")); + assert!( + value("authorization").is_some_and(|authorization| authorization + .starts_with("AWS4-HMAC-SHA256 Credential=TESTACCESSKEYID/")), + "should sign the request sent: {:?}", + value("authorization") + ); + assert_eq!(value("x-amz-content-sha256"), Some("UNSIGNED-PAYLOAD")); + }); + } + + #[test] + fn a_table_the_signer_cannot_use_is_refused_at_startup_and_at_deploy_validation() { + validate_settings_for_deploy_with(&settings_signing_the_origin(auth(None)), &[builder()]) + .expect("a table the signer can use should pass deploy validation"); + + // A blank credential is caught as an empty key reference when the + // deployment is validated, before the signer reads its table. + for (table, at_startup, at_deploy) in [ + ( + OriginAuth::new(SIGNER).with_setting("region", ""), + "[source.origin.auth] s3_sigv4 region must not be empty", + "[source.origin.auth] s3_sigv4 region must not be empty", + ), + ( + OriginAuth::new(SIGNER).with_setting("region", "Test_Region"), + "[source.origin.auth] s3_sigv4 region must contain only lowercase letters", + "[source.origin.auth] s3_sigv4 region must contain only lowercase letters", + ), + ( + auth(None).with_setting("access_key_id", " "), + "[source.origin.auth] s3_sigv4 credentials must not be empty", + "secret key reference at `source.origin.auth.access_key_id` must not be empty", + ), + ( + auth(None).with_setting("secret_access_key", ""), + "[source.origin.auth] s3_sigv4 credentials must not be empty", + "secret key reference at `source.origin.auth.secret_access_key` must not be empty", + ), + ( + auth(None).with_setting("bucket", "site"), + "unknown field `bucket`", + "unknown field `bucket`", + ), + ( + OriginAuth::new(SIGNER), + "missing field `region`", + "missing field `region`", + ), + ] { + let settings = settings_signing_the_origin(table.clone()); + let err = registry(&settings).expect_err("should refuse the table at startup"); + assert!( + format!("{err:?}").contains(at_startup), + "{table:?} should be refused saying `{at_startup}`: {err:?}" + ); + + let err = validate_settings_for_deploy_with(&settings, &[builder()]) + .expect_err("should refuse the table when the deployment is validated"); + assert!( + format!("{err:?}").contains(at_deploy), + "{table:?} should fail deploy validation saying `{at_deploy}`: {err:?}" + ); + } + } + + #[test] + fn an_asset_route_refusal_names_the_route() { + let mut settings = create_test_settings(); + let mut route = ProxyAssetRoute::new("/assets/", STORAGE_URL); + set_route_auth( + &mut route, + Some(OriginAuth::new(SIGNER).with_setting("region", "")), + ); + settings.proxy.asset_routes.push(route); + + let err = registry(&settings).expect_err("should refuse the route's table"); + assert!( + format!("{err:?}").contains( + "[proxy.asset_routes /assets/ source.origin.auth] s3_sigv4 region must not be empty" + ), + "{err:?}" + ); + } + + #[test] + fn the_signer_is_refused_where_the_build_does_not_offer_it() { + let settings = settings_signing_the_origin(auth(None)); + + let err = IntegrationRegistry::with_registrations(&settings, &[]) + .expect_err("should refuse a signer nothing offers"); + assert!( + format!("{err:?}").contains( + "[source.origin.auth] type `s3_sigv4` names an origin signer this deployment does not offer" + ), + "{err:?}" + ); + } + + #[test] + fn a_blank_session_token_is_no_token_and_a_deprecated_store_is_discarded() { + let settings = S3SigV4Settings::read( + &serde_json::from_value(json!({ + "region": " test-region-1 ", + "secret_store": "old-store", + "access_key_id": " TESTACCESSKEYID ", + "secret_access_key": "test-secret-access-key", + "session_token": " ", + })) + .expect("a table"), + "source.origin.auth", + ) + .expect("should read the table"); + + assert_eq!(settings.region, "test-region-1"); + assert_eq!(settings.access_key_id.expose(), "TESTACCESSKEYID"); + assert!(settings.session_token.is_none()); + assert!(settings.secret_store.is_none()); + } + + #[test] + fn the_documented_table_builds_the_signer() { + let toml = uncomment_block( + &template_with_resolved_required_secrets(), + "[source.origin.auth]", + ); + let settings = Settings::from_toml(&toml).expect("the documented table should parse"); + + // The template selects modules this crate's tests do not link, so + // the publisher's source is built on its own, as the registry builds + // it, rather than through a registry. + let location = settings + .origin_source() + .expect("the template should select the origin source") + .location( + &trusted_server_core::origin_auth::OriginSigners::new(vec![&S3_SIGV4]), + "source.origin", + ) + .expect("the documented table should build the signer"); + assert!( + location.auth().is_some(), + "the documented table should sign" + ); + } + + /// Every value this store hands back starts with this. + const CANARY: &str = "canary-"; + + /// Resolves every key name to a value of its own, long enough for any + /// minimum length a secret has. + struct CanaryStore; + + impl PlatformSecretStore for CanaryStore { + fn get_bytes( + &self, + _store_name: &StoreName, + key: &str, + ) -> Result, Report> { + Ok(format!("{CANARY}{key}-0123456789abcdef").into_bytes()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Ok(()) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Ok(()) + } + } + + /// Loads `settings` as a deployment does, with the signer's builder, so + /// its secret settings are looked up in the canary store. + fn load(settings: &Settings) -> Settings { + let data = serde_json::to_value(settings).expect("should serialize"); + let envelope = BlobEnvelope::new(data, "2026-01-01T00:00:00Z".to_owned()); + let envelope = serde_json::to_string(&envelope).expect("should serialize the envelope"); + settings_from_config_blob_with( + &envelope, + &CanaryStore, + &StoreName::from("ts_secrets"), + &[builder()], + ) + .expect("should load the settings") + } + + #[test] + fn the_credentials_are_looked_up_at_both_auth_paths_and_masked_on_the_configuration_page() { + let mut settings = create_test_settings(); + edit_origin(&mut settings, |origin| { + origin.url = STORAGE_URL.to_owned(); + origin.auth = Some( + OriginAuth::new(SIGNER) + .with_setting("region", REGION) + .with_setting("access_key_id", "origin-access") + .with_setting("secret_access_key", "origin-secret") + .with_setting("session_token", "origin-session"), + ); + }); + let mut route = ProxyAssetRoute::new("/assets/", "https://assets.example.com"); + set_route_auth( + &mut route, + Some(OriginAuth::new(SIGNER).with_setting("region", REGION)), + ); + settings.proxy.asset_routes.push(route); + + let loaded = load(&settings); + + let origin = loaded + .origin_source() + .and_then(|origin| origin.auth) + .expect("should keep the origin's auth"); + assert_eq!( + origin + .settings + .get("secret_access_key") + .and_then(Value::as_str), + Some("canary-origin-secret-0123456789abcdef") + ); + assert_eq!( + origin.settings.get("session_token").and_then(Value::as_str), + Some("canary-origin-session-0123456789abcdef") + ); + let route = loaded.proxy.asset_routes[0] + .origin_source() + .and_then(|origin| origin.auth) + .expect("should keep the route's auth"); + assert_eq!( + route.settings.get("access_key_id").and_then(Value::as_str), + Some("canary-access_key_id-0123456789abcdef"), + "a credential left out should be looked up under its own name" + ); + assert_eq!( + route + .settings + .get("secret_access_key") + .and_then(Value::as_str), + Some("canary-secret_access_key-0123456789abcdef") + ); + assert!( + !route.settings.contains_key("session_token"), + "a session token left out stays left out" + ); + + let view = build_view(&loaded).expect("should build the configuration view"); + let rendered = serde_json::to_string(&view.settings).expect("should serialize"); + assert!( + !rendered.contains(CANARY), + "no credential should reach the configuration page: {rendered}" + ); + for path in [ + "source.origin.auth.access_key_id", + "source.origin.auth.secret_access_key", + "source.origin.auth.session_token", + "proxy.asset_routes[0].source.origin.auth.access_key_id", + "proxy.asset_routes[0].source.origin.auth.secret_access_key", + ] { + assert!( + view.masked.iter().any(|masked| masked == path), + "{path} should be masked: {:?}", + view.masked + ); + } + assert_eq!( + view.settings["source"]["origin"]["auth"]["region"], + json!(REGION), + "the region is not a secret" + ); + assert_eq!( + view.settings["source"]["origin"]["auth"]["secret_access_key"], + json!(MASK) + ); + + let registry = registry(&loaded).expect("the loaded settings should build the signer"); + assert!(registry.publisher_source(&loaded).is_ok()); + } +} diff --git a/crates/trusted-server-core/src/s3_sigv4.rs b/crates/shared/aws/src/sigv4.rs similarity index 65% rename from crates/trusted-server-core/src/s3_sigv4.rs rename to crates/shared/aws/src/sigv4.rs index 921ed8d47f..90b9ec58de 100644 --- a/crates/trusted-server-core/src/s3_sigv4.rs +++ b/crates/shared/aws/src/sigv4.rs @@ -1,14 +1,13 @@ -//! Minimal AWS Signature Version 4 signing for `S3` asset requests. +//! AWS Signature Version 4 over the headers of a read from S3. //! -//! Asset routes use this module for read-only `S3` origins. The signer emits -//! header-based `SigV4` authentication with `UNSIGNED-PAYLOAD`, so it does not -//! need `AWS` SDK credential providers, request-body hashing, or presigned query -//! parameters. The caller must provide the final origin URL and `Host` header -//! after any path rewrite or query-strip policy has run. +//! The signer writes header-based authentication with `UNSIGNED-PAYLOAD`, so +//! it needs no credential provider, no request body and no presigned query. +//! The caller gives it the final address and `Host` header, after any path +//! rewrite or query policy has run, because the signature covers both. //! -//! Canonicalization follows the URL that will be sent to `S3`. Existing percent -//! escapes in path and query components are preserved and normalized to upper -//! case, while raw reserved bytes are encoded using `AWS` percent-encoding rules. +//! Canonicalization follows the address as it will be sent. A percent escape +//! already in the path or query is kept and written in upper case, and a raw +//! reserved byte is encoded the way the scheme requires. use std::time::SystemTime; @@ -17,25 +16,22 @@ use error_stack::Report; use hmac::{Hmac, Mac as _}; use http::{HeaderMap, HeaderValue, Method, header}; use sha2::{Digest as _, Sha256}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::redacted::Redacted; use url::Url; -use crate::error::TrustedServerError; -use crate::redacted::Redacted; - type HmacSha256 = Hmac; const ALGORITHM: &str = "AWS4-HMAC-SHA256"; const SERVICE: &str = "s3"; const TERMINATOR: &str = "aws4_request"; const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; -/// Prefix of every header name in this signing scheme's namespace. +/// Prefix of every header name in the scheme's own namespace. const NAMESPACE_PREFIX: &str = "x-amz-"; -/// Removes every header in this signing scheme's namespace. -/// -/// [`sign_headers`] replaces the few it writes. A caller that forwards a -/// reader's headers removes the rest first, so no header the reader chose can -/// change how the origin reads the signed request. +/// Removes every header in the scheme's namespace, so a header the reader +/// chose cannot change how the origin reads the signed request. +/// [`sign_headers`] then writes the few it needs. pub(crate) fn remove_namespace_headers(headers: &mut HeaderMap) { let names: Vec<_> = headers .keys() @@ -47,42 +43,27 @@ pub(crate) fn remove_namespace_headers(headers: &mut HeaderMap) { } } -/// `AWS` credentials used to sign an `S3` request. -/// -/// Values are loaded from the configured runtime secret store by the caller and -/// may be cached between signatures. Temporary credentials can include a session -/// token, which becomes the signed `x-amz-security-token` header. +/// The credentials a request is signed with. A session token, for temporary +/// credentials, becomes the signed `x-amz-security-token` header. #[derive(Debug, Clone)] -pub struct S3Credentials { - /// `AWS` access key ID. - pub access_key_id: String, - /// `AWS` secret access key. - pub secret_access_key: Redacted, - /// Optional `AWS` session token for temporary credentials. - pub session_token: Option>, +pub(crate) struct Credentials { + pub(crate) access_key_id: String, + pub(crate) secret_access_key: Redacted, + pub(crate) session_token: Option>, } -/// Sign an `S3` request header map using AWS Signature Version 4. -/// -/// The request URL and header map must already reflect the final origin request -/// that `S3` will receive, including path, query, and `Host` header. Existing -/// `Authorization` and `x-amz-*` signing headers are replaced so forwarded -/// client headers cannot influence the signature. -/// -/// This signer is scoped to read-only asset proxying. It always signs with -/// `x-amz-content-sha256: UNSIGNED-PAYLOAD`, which is valid for `GET` and -/// `HEAD` object reads and avoids buffering or hashing a request body. +/// Signs `headers` for `method` on `url` at `now`, replacing any earlier +/// `Authorization` and signing headers. /// /// # Errors /// -/// Returns a proxy error when a required signing header is missing or a signing -/// header value is invalid. -pub fn sign_headers( +/// When a header the signature needs is missing or cannot be written. +pub(crate) fn sign_headers( method: &Method, url: &Url, headers: &mut HeaderMap, region: &str, - credentials: &S3Credentials, + credentials: &Credentials, now: SystemTime, ) -> Result<(), Report> { let datetime = DateTime::::from(now); @@ -154,7 +135,7 @@ fn canonical_headers(headers: &HeaderMap) -> Result<(String, String), Report Result<(String, String), Report Result> { let value = value.to_str().map_err(|err| { Report::new(TrustedServerError::InvalidHeaderValue { - message: format!("S3 signing header value is not valid text: {err}"), + message: format!("signing header value is not valid text: {err}"), }) })?; Ok(value.split_whitespace().collect::>().join(" ")) } fn canonical_uri(url: &Url) -> String { - aws_percent_encode_preserving_escapes(url.path(), false) + percent_encode_preserving_escapes(url.path(), false) } fn canonical_query(url: &Url) -> String { @@ -189,8 +170,8 @@ fn canonical_query(url: &Url) -> String { .map(|part| { let (key, value) = part.split_once('=').unwrap_or((part, "")); ( - aws_percent_encode_preserving_escapes(key, true), - aws_percent_encode_preserving_escapes(value, true), + percent_encode_preserving_escapes(key, true), + percent_encode_preserving_escapes(value, true), ) }) .collect(); @@ -202,7 +183,7 @@ fn canonical_query(url: &Url) -> String { .join("&") } -fn aws_percent_encode_preserving_escapes(value: &str, encode_slash: bool) -> String { +fn percent_encode_preserving_escapes(value: &str, encode_slash: bool) -> String { let bytes = value.as_bytes(); let mut out = String::new(); let mut index = 0; @@ -218,13 +199,13 @@ fn aws_percent_encode_preserving_escapes(value: &str, encode_slash: bool) -> Str index += 3; continue; } - push_aws_percent_encoded_byte(&mut out, bytes[index], encode_slash); + push_percent_encoded_byte(&mut out, bytes[index], encode_slash); index += 1; } out } -fn push_aws_percent_encoded_byte(out: &mut String, byte: u8, encode_slash: bool) { +fn push_percent_encoded_byte(out: &mut String, byte: u8, encode_slash: bool) { match byte { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { out.push(char::from(byte)); @@ -262,7 +243,7 @@ impl HeaderValueResultExt for Result fn change_context_invalid_header(self, name: &str) -> Result> { self.map_err(|err| { Report::new(TrustedServerError::InvalidHeaderValue { - message: format!("invalid S3 signing header `{name}`: {err}"), + message: format!("invalid signing header `{name}`: {err}"), }) }) } @@ -272,8 +253,10 @@ impl HeaderValueResultExt for Result mod tests { use super::*; + /// The published example of a signed `GET`, whose host and credentials + /// are the ones the signature was published for. #[test] - fn signs_stable_s3_get_fixture() { + fn signs_the_published_get_example() { let url = Url::parse("https://examplebucket.s3.amazonaws.com/test.txt") .expect("should parse URL"); let mut headers = HeaderMap::new(); @@ -281,7 +264,7 @@ mod tests { header::HOST, HeaderValue::from_static("examplebucket.s3.amazonaws.com"), ); - let credentials = S3Credentials { + let credentials = Credentials { access_key_id: "AKIAIOSFODNN7EXAMPLE".to_string(), secret_access_key: Redacted::new( "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), @@ -319,9 +302,82 @@ mod tests { ); } + #[test] + fn a_session_token_is_signed_and_a_reader_s_header_in_the_namespace_is_removed() { + let url = Url::parse("https://bucket.storage.example.com/site/page").expect("url"); + let mut headers = HeaderMap::new(); + headers.insert( + header::HOST, + HeaderValue::from_static("bucket.storage.example.com"), + ); + headers.insert("x-amz-meta-reader", HeaderValue::from_static("chosen")); + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static("Bearer reader"), + ); + let credentials = Credentials { + access_key_id: "TESTACCESSKEYID".to_string(), + secret_access_key: Redacted::new("test-secret".to_string()), + session_token: Some(Redacted::new("test-session".to_string())), + }; + + remove_namespace_headers(&mut headers); + sign_headers( + &Method::HEAD, + &url, + &mut headers, + "test-region-1", + &credentials, + SystemTime::UNIX_EPOCH, + ) + .expect("should sign request"); + + assert!(headers.get("x-amz-meta-reader").is_none()); + assert_eq!( + headers.get("x-amz-security-token"), + Some(&HeaderValue::from_static("test-session")) + ); + let authorization = headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .expect("should set authorization header"); + assert!( + authorization.starts_with( + "AWS4-HMAC-SHA256 Credential=TESTACCESSKEYID/19700101/test-region-1/s3/aws4_request, \ + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, " + ), + "should sign the token with the other headers: {authorization}" + ); + } + + #[test] + fn a_request_without_a_host_header_is_refused() { + let url = Url::parse("https://bucket.storage.example.com/site/page").expect("url"); + let mut headers = HeaderMap::new(); + let credentials = Credentials { + access_key_id: "TESTACCESSKEYID".to_string(), + secret_access_key: Redacted::new("test-secret".to_string()), + session_token: None, + }; + + let err = sign_headers( + &Method::GET, + &url, + &mut headers, + "test-region-1", + &credentials, + SystemTime::UNIX_EPOCH, + ) + .expect_err("should refuse without a Host"); + assert!( + format!("{err:?}").contains("missing required signing header `host`"), + "{err:?}" + ); + } + #[test] fn credentials_debug_redacts_secret_material() { - let credentials = S3Credentials { + let credentials = Credentials { access_key_id: "AKIAIOSFODNN7EXAMPLE".to_string(), secret_access_key: Redacted::new("debug-secret-key".to_string()), session_token: Some(Redacted::new("debug-session-token".to_string())), @@ -349,7 +405,7 @@ mod tests { #[test] fn canonical_uri_preserves_existing_percent_encoded_path() { - let url = Url::parse("https://bucket.s3.us-east-1.amazonaws.com/foo%20bar/%e2%9c%93.jpg") + let url = Url::parse("https://bucket.storage.example.com/foo%20bar/%e2%9c%93.jpg") .expect("should parse URL"); assert_eq!(canonical_uri(&url), "/foo%20bar/%E2%9C%93.jpg"); @@ -357,7 +413,7 @@ mod tests { #[test] fn canonical_uri_encodes_raw_path_bytes_without_double_encoding() { - let url = Url::parse("https://bucket.s3.us-east-1.amazonaws.com/image*name%20raw.jpg") + let url = Url::parse("https://bucket.storage.example.com/image*name%20raw.jpg") .expect("should parse URL"); assert_eq!(canonical_uri(&url), "/image%2Aname%20raw.jpg"); @@ -365,15 +421,15 @@ mod tests { #[test] fn canonical_query_sorts_and_encodes() { - let url = Url::parse("https://bucket.s3.us-east-1.amazonaws.com/object?z=two&a=sp ace") + let url = Url::parse("https://bucket.storage.example.com/object?z=two&a=sp ace") .expect("should parse URL"); assert_eq!(canonical_query(&url), "a=sp%20ace&z=two"); } #[test] fn canonical_query_empty_query_is_empty() { - let url = Url::parse("https://bucket.s3.us-east-1.amazonaws.com/object?") - .expect("should parse URL"); + let url = + Url::parse("https://bucket.storage.example.com/object?").expect("should parse URL"); assert_eq!(canonical_query(&url), ""); } @@ -381,7 +437,7 @@ mod tests { #[test] fn canonical_query_preserves_plus_and_existing_escapes() { let url = Url::parse( - "https://bucket.s3.us-east-1.amazonaws.com/object?v=a+b&space=a%20b&slash=a/b&encoded=a%2bb&empty", + "https://bucket.storage.example.com/object?v=a+b&space=a%20b&slash=a/b&encoded=a%2bb&empty", ) .expect("should parse URL"); diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index f6c6f76bcc..23c62a0ff3 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -18,7 +18,7 @@ use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; use crate::integrations::IntegrationBuilder; -use crate::settings::{AssetOriginAuth, Ec, MODULE_IMPLEMENTATION_KEY, Settings}; +use crate::settings::{Ec, MODULE_IMPLEMENTATION_KEY, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; @@ -280,36 +280,9 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { true, ), ]; - // An asset route's source and the publisher's take the same signing - // table, so each holds the same three key references. - let origin_auth_keys = ["access_key_id", "secret_access_key", "session_token"]; - fields.extend(origin_auth_keys.map(|key| { - field( - vec![ - optional_object("proxy"), - optional_object("asset_routes"), - SecretPathSegment::ArrayEach, - optional_object("source"), - optional_object("origin"), - optional_object("auth"), - object(key), - ], - true, - ) - })); - fields.extend(origin_auth_keys.map(|key| { - field( - vec![ - optional_object("source"), - optional_object("origin"), - optional_object("auth"), - object(key), - ], - true, - ) - })); - // The settings a module declares in its own table, for core's - // modules and the ones a tool registered for deploy validation. + // The settings a module declares in its own table, and the ones an + // origin signer declares in its `auth` table, for core's modules and + // the ones a tool registered for deploy validation. let builders = crate::integrations::all_builders(deploy_integrations()).collect::>(); fields.extend(crate::module_secrets::secret_fields(&builders)); fields @@ -531,38 +504,13 @@ fn validate_secret_key_references( } } - for (index, route) in settings.proxy.asset_routes.iter().enumerate() { - if let Some(auth) = route.origin_source().and_then(|origin| origin.auth) { - validate_origin_auth_key_references( - &format!("proxy.asset_routes[{index}].source.origin.auth"), - &auth, - )?; - } - } - if let Some(auth) = settings.origin_source().and_then(|origin| origin.auth) { - validate_origin_auth_key_references("source.origin.auth", &auth)?; - } - - Ok(()) -} + // An origin signer's secret settings are known through the builders that + // offer the signer, as they are when the settings load. + let builders: Vec = + crate::integrations::all_builders(extra_integrations).collect(); + let signers = crate::origin_auth::OriginSigners::offered_by(&builders)?; + crate::origin_auth::check_key_references(settings, &signers)?; -/// Refuses an empty secret key reference in an origin auth table at `path`. -fn validate_origin_auth_key_references( - path: &str, - auth: &AssetOriginAuth, -) -> Result<(), Report> { - let AssetOriginAuth::S3SigV4(auth) = auth; - validate_secret_key_reference( - &format!("{path}.access_key_id"), - auth.access_key_id.expose(), - )?; - validate_secret_key_reference( - &format!("{path}.secret_access_key"), - auth.secret_access_key.expose(), - )?; - if let Some(token) = &auth.session_token { - validate_secret_key_reference(&format!("{path}.session_token"), token.expose())?; - } Ok(()) } @@ -598,15 +546,13 @@ mod tests { use super::*; use crate::integrations::IntegrationRegistration; use crate::integrations::js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID; + use crate::origin_auth::fixture; use crate::redacted::Redacted; - use crate::settings::{ - AnalyticsConfig, ProxyAssetRoute, S3SigV4AuthConfig, TinybirdSettings, - TrustedClientIpConfig, - }; + use crate::settings::{AnalyticsConfig, TinybirdSettings, TrustedClientIpConfig}; use crate::test_support::template::{template_with_resolved_required_secrets, uncomment_block}; use crate::test_support::tests::{ crate_test_settings_str, crate_test_settings_str_with_ec_section, edit_origin, - select_hmac_module, set_route_auth, + select_hmac_module, }; use edgezero_core::app_config::AppConfigMeta; use edgezero_core::blob_envelope::BlobEnvelope; @@ -1076,19 +1022,6 @@ formats = [{ width = 300, height = 250 }] } } - #[test] - fn omitted_s3_secret_references_materialize_as_defaults() { - let auth: S3SigV4AuthConfig = - toml::from_str("region = \"us-east-1\"").expect("should apply S3 secret defaults"); - - assert_eq!(auth.access_key_id.expose(), "access_key_id"); - assert_eq!(auth.secret_access_key.expose(), "secret_access_key"); - - let serialized = serde_json::to_value(auth).expect("should serialize S3 auth"); - assert_eq!(serialized["access_key_id"], "access_key_id"); - assert_eq!(serialized["secret_access_key"], "secret_access_key"); - } - /// An `[analytics]` section selecting the analytics module with `settings`. fn analytics_selecting(settings: TinybirdSettings) -> AnalyticsConfig { AnalyticsConfig { @@ -1104,34 +1037,16 @@ formats = [{ width = 300, height = 250 }] secret_store: Some("legacy-tinybird-store".to_string()), ..TinybirdSettings::default() })); - let mut route = ProxyAssetRoute::new( - "/assets/", - "https://examplebucket.s3.us-east-1.amazonaws.com", - ); - set_route_auth( - &mut route, - Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "us-east-1".to_string(), - secret_store: Some("legacy-s3-store".to_string()), - access_key_id: Redacted::new("s3-access-key".to_string()), - secret_access_key: Redacted::new("s3-secret-key".to_string()), - session_token: None, - origin_query: None, - })), - ); - settings.proxy.asset_routes.push(route); settings .normalize_deserialized() .expect("should normalize settings without key collisions"); let serialized = serde_json::to_string(&settings).expect("should serialize settings"); - for legacy_store in ["legacy-tinybird-store", "legacy-s3-store"] { - assert!( - !serialized.contains(legacy_store), - "serialized config should omit deprecated selector {legacy_store}" - ); - } + assert!( + !serialized.contains("legacy-tinybird-store"), + "serialized config should omit the deprecated selector" + ); } #[test] @@ -1411,36 +1326,27 @@ passphrase = "production-secret-key-32-bytes-min" ); } - /// Valid settings whose `[source.origin]` signs its requests. - fn settings_signing_the_publisher_origin( - access_key_id: &str, - secret_access_key: &str, - session_token: Option<&str>, - ) -> Settings { + /// Valid settings whose `[source.origin]` signs its requests through the + /// fixture signer, naming `key_name` and, when given, `token_name`. + fn settings_signing_the_publisher_origin(key_name: &str, token_name: Option<&str>) -> Settings { let mut settings = valid_settings(); edit_origin(&mut settings, |origin| { - origin.url = "https://storage.example.com/bucket/site".to_string() - }); - edit_origin(&mut settings, |origin| { - origin.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "test-region-1".to_string(), - secret_store: None, - access_key_id: Redacted::new(access_key_id.to_string()), - secret_access_key: Redacted::new(secret_access_key.to_string()), - session_token: session_token.map(|token| Redacted::new(token.to_string())), - origin_query: None, - })) + origin.url = "https://storage.example.com/bucket/site".to_string(); + let mut auth = fixture::auth("site").with_setting("key_name", key_name); + if let Some(token_name) = token_name { + auth = auth.with_setting("token_name", token_name); + } + origin.auth = Some(auth); }); settings } #[test] fn deploy_validation_accepts_a_source_origin_auth_naming_its_keys() { - validate_settings_for_deploy(&settings_signing_the_publisher_origin( - "origin_access_key_id", - "origin_secret_access_key", - Some("origin_session_token"), - )) + validate_settings_for_deploy_with( + &settings_signing_the_publisher_origin("origin_key", Some("origin_token")), + &[fixture::builder()], + ) .expect("should accept an auth whose key references are named"); } @@ -1448,15 +1354,15 @@ passphrase = "production-secret-key-32-bytes-min" fn deploy_validation_refuses_an_empty_source_origin_auth_key_reference() { for (settings, path) in [ ( - settings_signing_the_publisher_origin(" ", "origin_secret_access_key", None), - "source.origin.auth.access_key_id", + settings_signing_the_publisher_origin(" ", None), + "source.origin.auth.key_name", ), ( - settings_signing_the_publisher_origin("origin_access_key_id", "", None), - "source.origin.auth.secret_access_key", + settings_signing_the_publisher_origin("origin_key", Some("")), + "source.origin.auth.token_name", ), ] { - let err = validate_settings_for_deploy(&settings) + let err = validate_settings_for_deploy_with(&settings, &[fixture::builder()]) .expect_err("should refuse an empty auth key reference"); assert!( format!("{err:?}").contains(&format!( @@ -1468,20 +1374,9 @@ passphrase = "production-secret-key-32-bytes-min" } #[test] - fn a_blank_source_origin_session_token_is_read_as_none() { - let settings = settings_signing_the_publisher_origin( - "origin_access_key_id", - "origin_secret_access_key", - Some(" "), - ); - validate_settings_for_deploy(&settings) - .expect("a blank session token is no token, as a document's is"); - let Some(AssetOriginAuth::S3SigV4(auth)) = - settings.origin_source().and_then(|origin| origin.auth) - else { - panic!("should keep the origin auth"); - }; - assert!(auth.session_token.is_none()); + fn deploy_validation_leaves_a_signer_s_settings_alone_without_its_builder() { + validate_settings_for_deploy(&settings_signing_the_publisher_origin(" ", None)) + .expect("the key name is the signer's business, and no builder offers it here"); } #[test] @@ -1501,10 +1396,7 @@ passphrase = "production-secret-key-32-bytes-min" .origin_source() .expect("the template should select the origin source"); - assert!( - matches!(origin.auth, Some(AssetOriginAuth::S3SigV4(_))), - "the documented auth should be read" - ); + assert!(origin.auth.is_some(), "the documented auth should be read"); assert_eq!( origin.directory_index.as_deref(), Some("index.html"), diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index b1baa3a518..e38aeb38bd 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -90,6 +90,12 @@ pub fn settings_from_config_blob_with( let mut data = envelope.into_data(); remove_inactive_secret_references(&mut data); crate::module_secrets::clear_unused(&mut data, &builders); + // A signer's secret setting left out takes its default key name, so the + // lookup below finds it as a written one. + crate::origin_auth::fill_secret_defaults( + &mut data, + &crate::origin_auth::OriginSigners::offered_by(&builders)?, + ); let resolved = resolve_secret_references_with::( &mut data, secret_store, @@ -139,11 +145,11 @@ fn json_bool_or_string_is_true(value: Option<&serde_json::Value>) -> bool { mod tests { use super::*; use crate::ec::module::{HMAC_MODULE_KEY, HOST_SIGNALS_MODULE_KEY}; + use crate::origin_auth::fixture; use crate::platform::{PlatformError, StoreId}; use crate::redacted::Redacted; use crate::settings::{ - AnalyticsConfig, AssetOriginAuth, EcPartner, ProxyAssetRoute, S3SigV4AuthConfig, - TinybirdSettings, TrustedClientIpConfig, + AnalyticsConfig, EcPartner, ProxyAssetRoute, TinybirdSettings, TrustedClientIpConfig, }; use crate::test_support::tests::{ crate_test_settings_str, edit_origin, hmac_passphrase, select_hmac_module, @@ -201,9 +207,8 @@ mod tests { let value = match key { "unit-test-proxy-secret" => "unit-test-proxy-secret-32-bytes-ok", "tinybird-token-key" => "resolved-tinybird-token", - "access_key_id" | "s3-access-key" => "AKIAIOSFODNN7EXAMPLE", - "secret_access_key" | "s3-secret-key" => "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - "s3-session-key" => "resolved-session-token", + "fixture_key" | "signer-key" => "resolved-signer-key", + "signer-token" => "resolved-signer-token", "partner-api-token-key" => "resolved-partner-api-token-32-bytes-ok", "partner-pull-token-key" => "resolved-partner-pull-token-32-bytes-ok", "trusted-client-ip-key" => "resolved-trusted-client-ip-secret-32-bytes", @@ -518,20 +523,14 @@ mod tests { ..TinybirdSettings::default() }), }); - let mut route = ProxyAssetRoute::new( - "/assets/", - "https://examplebucket.s3.us-east-1.amazonaws.com", - ); + let mut route = ProxyAssetRoute::new("/assets/", "https://assets.example.com"); set_route_auth( &mut route, - Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "us-east-1".to_string(), - secret_store: Some("legacy-s3-store".to_string()), - access_key_id: Redacted::new("s3-access-key".to_string()), - secret_access_key: Redacted::new("s3-secret-key".to_string()), - session_token: Some(Redacted::new("s3-session-key".to_string())), - origin_query: None, - })), + Some( + fixture::auth("assets") + .with_setting("key_name", "signer-key") + .with_setting("token_name", "signer-token"), + ), ); original.proxy.asset_routes.push(route); original @@ -539,10 +538,11 @@ mod tests { .partners .push(partner_with_pull_sync(true, "partner-pull-token-key")); - let reconstructed = settings_from_config_blob( + let reconstructed = settings_from_config_blob_with( &envelope_json(&original), &UnifiedSecretStore, &StoreName::from("ts_secrets"), + &[fixture::builder()], ) .expect("should resolve every static credential from the mapped store"); @@ -557,21 +557,19 @@ mod tests { let auth = reconstructed.proxy.asset_routes[0] .origin_source() .and_then(|origin| origin.auth) - .expect("should preserve S3 auth"); - let AssetOriginAuth::S3SigV4(auth) = auth; - assert_eq!(auth.access_key_id.expose(), "AKIAIOSFODNN7EXAMPLE"); + .expect("should preserve the route's auth"); assert_eq!( - auth.secret_access_key.expose(), - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + auth.settings + .get("key_name") + .and_then(serde_json::Value::as_str), + Some("resolved-signer-key") ); assert_eq!( - auth.session_token - .as_ref() - .map(Redacted::expose) - .map(String::as_str), - Some("resolved-session-token") + auth.settings + .get("token_name") + .and_then(serde_json::Value::as_str), + Some("resolved-signer-token") ); - assert!(auth.secret_store.is_none()); assert_eq!( reconstructed.ec.partners[0] .api_token @@ -750,78 +748,107 @@ mod tests { } #[test] - fn omitted_s3_secret_references_resolve_default_store_keys() { + fn a_signer_secret_setting_left_out_resolves_its_default_key_name() { let mut original = test_settings(); - let mut route = ProxyAssetRoute::new( - "/default-s3/", - "https://examplebucket.s3.us-east-1.amazonaws.com", - ); - set_route_auth( - &mut route, - Some(AssetOriginAuth::S3SigV4( - toml::from_str("region = \"us-east-1\"").expect("should apply S3 secret defaults"), - )), - ); + let mut route = ProxyAssetRoute::new("/default-signed/", "https://assets.example.com"); + set_route_auth(&mut route, Some(fixture::auth("assets"))); original.proxy.asset_routes.push(route); - let reconstructed = settings_from_config_blob( + let reconstructed = settings_from_config_blob_with( &envelope_json(&original), &UnifiedSecretStore, &StoreName::from("ts_secrets"), + &[fixture::builder()], ) - .expect("should resolve default S3 secret keys"); + .expect("should resolve the default key name"); - let AssetOriginAuth::S3SigV4(auth) = reconstructed.proxy.asset_routes[0] + let auth = reconstructed.proxy.asset_routes[0] .origin_source() .and_then(|origin| origin.auth) - .expect("should preserve S3 auth"); - assert_eq!(auth.access_key_id.expose(), "AKIAIOSFODNN7EXAMPLE"); + .expect("should preserve the route's auth"); assert_eq!( - auth.secret_access_key.expose(), - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + auth.settings + .get("key_name") + .and_then(serde_json::Value::as_str), + Some("resolved-signer-key"), + "the default key name should be looked up" + ); + assert!( + !auth.settings.contains_key("token_name"), + "a setting with no default should stay left out" ); } #[test] - fn source_origin_auth_resolves_its_credentials_from_the_store() { + fn source_origin_auth_resolves_its_secret_settings_from_the_store() { let mut original = test_settings(); edit_origin(&mut original, |origin| { - origin.url = "https://storage.example.com/bucket/site".to_string() - }); - edit_origin(&mut original, |origin| { - origin.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "test-region-1".to_string(), - secret_store: None, - access_key_id: Redacted::new("s3-access-key".to_string()), - secret_access_key: Redacted::new("s3-secret-key".to_string()), - session_token: Some(Redacted::new("s3-session-key".to_string())), - origin_query: None, - })) + origin.url = "https://storage.example.com/bucket/site".to_string(); + origin.auth = Some( + fixture::auth("site") + .with_setting("key_name", "signer-key") + .with_setting("token_name", "signer-token"), + ); }); - let reconstructed = settings_from_config_blob( + let reconstructed = settings_from_config_blob_with( &envelope_json(&original), &UnifiedSecretStore, &StoreName::from("ts_secrets"), + &[fixture::builder()], ) .expect("should resolve the publisher origin credentials from the mapped store"); - let Some(AssetOriginAuth::S3SigV4(auth)) = - reconstructed.origin_source().and_then(|origin| origin.auth) - else { - panic!("should keep the origin auth"); - }; - assert_eq!(auth.access_key_id.expose(), "AKIAIOSFODNN7EXAMPLE"); + let auth = reconstructed + .origin_source() + .and_then(|origin| origin.auth) + .expect("should keep the origin auth"); assert_eq!( - auth.secret_access_key.expose(), - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + auth.settings + .get("key_name") + .and_then(serde_json::Value::as_str), + Some("resolved-signer-key") ); assert_eq!( - auth.session_token - .as_ref() - .map(Redacted::expose) - .map(String::as_str), - Some("resolved-session-token") + auth.settings + .get("token_name") + .and_then(serde_json::Value::as_str), + Some("resolved-signer-token") + ); + assert_eq!( + auth.settings + .get("realm") + .and_then(serde_json::Value::as_str), + Some("site"), + "a setting that is not a secret should be left as written" + ); + } + + #[test] + fn a_signer_s_secret_settings_are_not_looked_up_without_its_builder() { + let mut original = test_settings(); + edit_origin(&mut original, |origin| { + origin.url = "https://storage.example.com/bucket/site".to_string(); + origin.auth = Some(fixture::auth("site").with_setting("key_name", "signer-key")); + }); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should load without the signer's builder"); + + let auth = reconstructed + .origin_source() + .and_then(|origin| origin.auth) + .expect("should keep the origin auth"); + assert_eq!( + auth.settings + .get("key_name") + .and_then(serde_json::Value::as_str), + Some("signer-key"), + "the key name should be left as written" ); } diff --git a/crates/trusted-server-core/src/host_header.rs b/crates/trusted-server-core/src/host_header.rs index 8bff832399..1dc117d758 100644 --- a/crates/trusted-server-core/src/host_header.rs +++ b/crates/trusted-server-core/src/host_header.rs @@ -2,7 +2,7 @@ //! //! These helpers intentionally validate only configuration values. Request input //! must not flow into this module because outbound `Host` controls origin -//! routing and SigV4-style signing decisions. +//! routing and request signing. use std::net::{Ipv4Addr, Ipv6Addr}; diff --git a/crates/trusted-server-core/src/inspect/config/tests.rs b/crates/trusted-server-core/src/inspect/config/tests.rs index 4b19b49cea..d4b4bfd5d5 100644 --- a/crates/trusted-server-core/src/inspect/config/tests.rs +++ b/crates/trusted-server-core/src/inspect/config/tests.rs @@ -5,6 +5,7 @@ use edgezero_core::blob_envelope::BlobEnvelope; use super::*; use crate::config_payload::settings_from_config_blob_with; use crate::integrations::{IntegrationBuilder, ModuleSecretSetting}; +use crate::origin_auth::fixture; use crate::platform::{PlatformError, PlatformSecretStore, StoreId, StoreName}; use crate::secret_resolution::ResolvedSecrets; use crate::settings::ProxyAssetRoute; @@ -204,16 +205,16 @@ fn a_key_the_inspect_section_does_not_have_is_refused_naming_it() { } /// Documents that between them hold a secret at every path core declares -/// one, each a key name the canary store resolves to a value of its own. +/// one, and at both paths the fixture signer declares its own, each a key +/// name the canary store resolves to a value of its own. fn documents_holding_every_declared_secret() -> Vec<(&'static str, Value)> { let mut main = document(""); main["publisher"]["proxy_secret"] = json!("proxy"); main["source"]["origin"]["auth"] = json!({ - "type": "s3_sigv4", - "region": "test-region-1", - "access_key_id": "origin-access", - "secret_access_key": "origin-secret", - "session_token": "origin-session", + "type": fixture::SIGNER, + "realm": "site", + "key_name": "origin-key", + "token_name": "origin-token", }); main["ec"]["hmac"]["passphrase"] = json!("hmac"); main["ec"]["partners"] = json!([{ @@ -237,11 +238,10 @@ fn documents_holding_every_declared_secret() -> Vec<(&'static str, Value)> { "origin": { "url": "https://assets.example.com", "auth": { - "type": "s3_sigv4", - "region": "us-east-1", - "access_key_id": "route-access", - "secret_access_key": "route-secret", - "session_token": "route-session", + "type": fixture::SIGNER, + "realm": "assets", + "key_name": "route-key", + "token_name": "route-token", }, }, }, @@ -301,14 +301,16 @@ fn generalized(path: &[PathStep]) -> String { out } -/// Every secret core declares, loaded the way a deployment loads it, is -/// live in the settings and absent from both forms the endpoint serves, -/// and wherever the view still has its path, that path is masked. +/// Every secret core declares, and every one the fixture signer declares at +/// the `auth` paths, loaded the way a deployment loads it, is live in the +/// settings and absent from both forms the endpoint serves, and wherever the +/// view still has its path, that path is masked. #[test] fn every_declared_core_secret_is_masked_and_never_served() { let mut covered = BTreeSet::new(); for (name, data) in documents_holding_every_declared_secret() { - let settings = load(data).unwrap_or_else(|error| panic!("{name}: {error:?}")); + let settings = load_with(data, &[fixture::builder()]) + .unwrap_or_else(|error| panic!("{name}: {error:?}")); let record = settings.resolved_secrets(); let normal = serde_json::to_string(&settings).expect("should serialize"); assert!( @@ -337,6 +339,7 @@ fn every_declared_core_secret_is_masked_and_never_served() { let declared: BTreeSet = TrustedServerAppConfig::secret_fields() .iter() + .chain(crate::module_secrets::secret_fields(&[fixture::builder()]).iter()) .filter_map(|field| PathPattern::from_segments(&field.path)) .map(|pattern| pattern.as_str().to_owned()) .collect(); @@ -345,6 +348,17 @@ fn every_declared_core_secret_is_masked_and_never_served() { missing.is_empty(), "every declared secret should be exercised, missing {missing:?}" ); + for path in [ + "source.origin.auth.key_name", + "source.origin.auth.token_name", + "proxy.asset_routes[].source.origin.auth.key_name", + "proxy.asset_routes[].source.origin.auth.token_name", + ] { + assert!( + covered.contains(path), + "the signer's secret at {path} should be declared and masked" + ); + } } /// Test settings holding a value of its own in every field marked @@ -591,10 +605,9 @@ const ASSET_ROUTES: &str = r#" source.origin.url = "https://assets.example.com" [proxy.asset_routes.source.origin.auth] - type = "s3_sigv4" - region = "us-east-1" - access_key_id = "route-access" - secret_access_key = "route-secret" + type = "fixture_scheme" + realm = "assets" + key_name = "route-key" [[proxy.asset_routes]] prefix = "/media/" @@ -645,8 +658,8 @@ fn a_pattern_not_honored_as_written_is_refused_saying_why() { "show names `publisher.proxy_secret`, which is a secret", ), ( - r#"show = ["proxy.asset_routes[].source.origin.auth.secret_access_key"]"#, - "show names `proxy.asset_routes[].source.origin.auth.secret_access_key`, which is a secret", + r#"show = ["proxy.asset_routes[].source.origin.auth.key_name"]"#, + "show names `proxy.asset_routes[].source.origin.auth.key_name`, which is a secret", ), ( r#"hide = ["publisher.no_such_setting"]"#, @@ -692,10 +705,14 @@ fn a_pattern_not_honored_as_written_is_refused_saying_why() { "config = false publishes no page, so they change nothing", ), ] { + // The route's signer is on offer, so its key name is a declared secret. let message = format!( "{:?}", - load(document(&format!("{ASSET_ROUTES}\n[inspect]\n{section}"))) - .expect_err("the document should be refused") + load_with( + document(&format!("{ASSET_ROUTES}\n[inspect]\n{section}")), + &[fixture::builder()], + ) + .expect_err("the document should be refused") ); assert!(message.contains(reason), "{section}: {message}"); diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 16c3d3e8f5..68a90d915b 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -12,6 +12,7 @@ use url::Url; use crate::auction::AuctionPlan; use crate::auction::demand::{AdServerImplementation, DemandImplementation}; use crate::error::TrustedServerError; +use crate::origin_auth::OriginSignerModule; use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, RuntimeServices}; use crate::settings::Settings; @@ -426,6 +427,7 @@ pub struct IntegrationBuilder { loading_cache: Option, shared: Option<&'static str>, cache_features: &'static [&'static str], + origin_signer: Option<&'static OriginSignerModule>, } /// Gives the loading cache a module supplies, see @@ -473,6 +475,7 @@ impl IntegrationBuilder { loading_cache: None, shared: None, cache_features: &[], + origin_signer: None, } } @@ -503,6 +506,7 @@ impl IntegrationBuilder { loading_cache: None, shared: None, cache_features: &[], + origin_signer: None, } } @@ -680,6 +684,24 @@ impl IntegrationBuilder { self.cache_features } + /// Offers the origin signer this crate supplies, which a source's `auth` + /// table selects with `type = ""`, see [`crate::origin_auth`]. + /// + /// The offer is not a selection. The signer is built where a source + /// names it, and its secret settings are looked up as the settings load + /// whatever the sections select. + #[must_use] + pub const fn with_origin_signer(mut self, signer: &'static OriginSignerModule) -> Self { + self.origin_signer = Some(signer); + self + } + + /// The origin signer this builder offers, when it offers one. + #[must_use] + pub const fn origin_signer(&self) -> Option<&'static OriginSignerModule> { + self.origin_signer + } + /// The loading cache this builder offers, when it offers one. #[must_use] pub const fn loading_cache(&self) -> Option { diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 56d0fd9737..93fe76ee6e 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -21,6 +21,7 @@ use crate::geo::GeoInfo; use crate::http_util::is_navigation_request; use crate::middleware::{Middleware, MiddlewareChain, MiddlewarePhase, PhaseEntries}; use crate::module_context::{ModuleCall, ModuleContext, ModuleRequest, ResolvedRequest}; +use crate::origin_auth::OriginSigners; use crate::platform::{DisabledGeo, GeoModule, PlatformGeoModule, RuntimeServices}; use crate::prefix::{PrefixModule, Prefixes, SelectedPrefixModule}; use crate::robots_txt::{ALLOW_ALL, REFUSE_ALL, RobotsTxtContributor}; @@ -964,6 +965,8 @@ struct IntegrationRegistryInner { prefix_module: SelectedPrefixModule, // The source modules the running registrations supply, each with its name. source_modules: Vec<(&'static str, Arc)>, + // The origin signers the builders offer, which a source's `auth` names. + origin_signers: OriginSigners, // The publisher's source and each asset route's, in route order, built // from the settings. `None` for a registry built without settings. sources: Option, @@ -1011,6 +1014,7 @@ impl Default for IntegrationRegistryInner { prefix_modules: Vec::new(), prefix_module: SelectedPrefixModule::default(), source_modules: Vec::new(), + origin_signers: OriginSigners::default(), sources: None, } } @@ -1671,11 +1675,12 @@ fn check_section_selections( /// The source `selection` names, built from its table, where `setting` names /// the table in a refusal. Core's `origin` is always on offer, beside the -/// sources the registrations supply. +/// sources the registrations supply, and signs through `signers`. fn build_source( selection: &crate::provider_table::SectionModules, setting: &str, offered: &[(&'static str, Arc)], + signers: &OriginSigners, ) -> Result, Report> { let (written, table) = crate::source::selected(selection, setting)?; let table_setting = format!("{setting}.{written}"); @@ -1699,7 +1704,7 @@ fn build_source( })); }; if found == crate::source::ORIGIN_MODULE { - return crate::source::OriginFactory.build(&table, &table_setting); + return crate::source::OriginFactory::new(signers.clone()).build(&table, &table_setting); } let factory = offered .iter() @@ -1717,13 +1722,19 @@ fn build_source( fn build_sources( settings: &Settings, offered: &[(&'static str, Arc)], + signers: &OriginSigners, ) -> Result> { - let publisher = build_source(&settings.source, crate::source::SOURCE_SECTION, offered)?; + let publisher = build_source( + &settings.source, + crate::source::SOURCE_SECTION, + offered, + signers, + )?; let routes = settings .proxy .asset_routes .iter() - .map(|route| build_source(&route.source, &route.source_setting(), offered)) + .map(|route| build_source(&route.source, &route.source_setting(), offered, signers)) .collect::, _>>()?; Ok(BuiltSources { publisher, routes }) } @@ -1743,7 +1754,12 @@ impl IntegrationRegistry { ) -> Result, Report> { match &self.inner.sources { Some(sources) => Ok(Arc::clone(&sources.publisher)), - None => build_source(&settings.source, crate::source::SOURCE_SECTION, &[]), + None => build_source( + &settings.source, + crate::source::SOURCE_SECTION, + &[], + &OriginSigners::default(), + ), } } @@ -1771,7 +1787,12 @@ impl IntegrationRegistry { message: format!("there is no asset route at {index}"), }) })?; - build_source(&route.source, &route.source_setting(), &[]) + build_source( + &route.source, + &route.source_setting(), + &[], + &OriginSigners::default(), + ) } /// Tells the registry what the adapter reports about the platform, which @@ -1896,6 +1917,12 @@ impl IntegrationRegistry { })); } + // Which signers a source's `auth` can name is known from the builders + // alone, so two offering one name are refused before anything runs. + inner.origin_signers = OriginSigners::offered_by( + &crate::integrations::all_builders(extra).collect::>(), + )?; + for builder in crate::integrations::all_builders(extra) { if let Some((_, first_source)) = inner.builder_ids.iter().find(|(id, _)| *id == builder.id()) @@ -2170,7 +2197,11 @@ impl IntegrationRegistry { inner.ec_module = resolve_ec_module(settings, &inner); inner.device_module = resolve_device_module(settings, &inner)?; inner.prefix_module = resolve_prefix_module(settings, &inner)?; - inner.sources = Some(build_sources(settings, &inner.source_modules)?); + inner.sources = Some(build_sources( + settings, + &inner.source_modules, + &inner.origin_signers, + )?); inner.geo_module = geo_module; check_robots_txt_selection(settings, &inner)?; if let Some(config) = settings.robots_txt.as_ref() { @@ -2783,6 +2814,7 @@ impl IntegrationRegistry { prefix_modules: Vec::new(), prefix_module: SelectedPrefixModule::default(), source_modules: Vec::new(), + origin_signers: OriginSigners::default(), sources: None, }), plan: None, @@ -2867,6 +2899,7 @@ impl IntegrationRegistry { prefix_modules: Vec::new(), prefix_module: SelectedPrefixModule::default(), source_modules: Vec::new(), + origin_signers: OriginSigners::default(), sources: None, }), plan: None, diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 3a84b26c25..e3bdf6ba7d 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -61,6 +61,7 @@ pub mod module_context; pub mod module_name; pub(crate) mod module_secrets; pub mod openrtb; +pub mod origin_auth; pub mod permission_signal; pub mod permissions; pub mod platform; @@ -74,7 +75,6 @@ pub mod request_signing; pub mod response_privacy; pub mod robots_txt; pub mod rsc_flight; -pub(crate) mod s3_sigv4; pub mod secret_resolution; pub mod service_version; pub mod settings; diff --git a/crates/trusted-server-core/src/module_secrets.rs b/crates/trusted-server-core/src/module_secrets.rs index 24e9653fc6..fae82a472e 100644 --- a/crates/trusted-server-core/src/module_secrets.rs +++ b/crates/trusted-server-core/src/module_secrets.rs @@ -128,14 +128,21 @@ pub(crate) fn clear_unused(data: &mut Value, builders: &[IntegrationBuilder]) { } /// The secret leaves the builders' modules declare, at each name a module's -/// table can be written under. +/// table can be written under, and the ones their origin signers declare, at +/// each place an `auth` table can be written. /// /// Every level is optional, so a module that is not configured, or a setting /// [`clear_unused`] removed, is passed over. #[must_use] pub(crate) fn secret_fields(builders: &[IntegrationBuilder]) -> Vec { let optional = |name: &'static str| SecretPathSegment::OptionalField(Cow::Borrowed(name)); - let mut fields = Vec::new(); + let signers = crate::origin_auth::OriginSigners::new( + builders + .iter() + .filter_map(IntegrationBuilder::origin_signer) + .collect(), + ); + let mut fields = crate::origin_auth::secret_fields(&signers); for builder in builders { let Some((section, forms)) = written_forms(builder) else { continue; diff --git a/crates/trusted-server-core/src/origin_auth.rs b/crates/trusted-server-core/src/origin_auth.rs new file mode 100644 index 0000000000..a4457dc836 --- /dev/null +++ b/crates/trusted-server-core/src/origin_auth.rs @@ -0,0 +1,943 @@ +//! Signing of requests to an origin that answers only signed reads. +//! +//! A source's `auth` table selects a signer by `type` and holds the signer's +//! settings beside it: +//! +//! ```toml +//! [source.origin] +//! url = "https://storage.example.com/bucket/site" +//! auth = { type = "example_scheme", origin_query = "strip", key_name = "origin_key" } +//! ``` +//! +//! Core reads `type`, which names a signer, and `origin_query`, which says +//! whether the reader's query is left off a signed request. The rest of the +//! table belongs to the signer. A crate offers a signer on its builder with +//! [`IntegrationBuilder::with_origin_signer`](crate::integrations::IntegrationBuilder::with_origin_signer), +//! and the sources are built from the settings when the registry is, which +//! is where a `type` nothing offers, or a table the signer cannot use, is +//! refused. +//! +//! A signer declares which of its settings hold the name of a secret, so +//! each is looked up in the default secret store as the settings load, at +//! both places an `auth` table can be written, and masked on the +//! configuration page. + +use std::borrow::Cow; +use std::fmt; +use std::sync::Arc; +use std::time::SystemTime; + +use edgezero_core::app_config::{SecretField, SecretKind, SecretPathSegment}; +use error_stack::Report; +use http::{HeaderMap, Method}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use url::Url; + +use crate::error::TrustedServerError; +use crate::integrations::IntegrationBuilder; +use crate::settings::{OriginQueryPolicy, Settings}; + +/// An `auth` table as a source writes it: the signer it names, whether the +/// reader's query is kept, and the signer's own settings. +#[derive(Clone, Default, Deserialize, Serialize, PartialEq)] +pub struct OriginAuth { + /// The name of the signer, as a crate offers it. + #[serde(rename = "type")] + pub signer: String, + /// Whether the reader's query is kept on the signed request. Left unset, + /// the route's own default applies. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin_query: Option, + /// The signer's settings, which core does not read. + #[serde(flatten)] + pub settings: Map, +} + +impl fmt::Debug for OriginAuth { + // The signer's settings hold its secrets once the settings have loaded, + // so only their names are shown. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OriginAuth") + .field("signer", &self.signer) + .field("origin_query", &self.origin_query) + .field("settings", &self.settings.keys().collect::>()) + .finish() + } +} + +impl OriginAuth { + /// An `auth` table selecting `signer` with no settings. + #[must_use] + pub fn new(signer: impl Into) -> Self { + Self { + signer: signer.into(), + ..Self::default() + } + } + + /// The same table with `key` set to `value` among the signer's settings. + #[must_use] + pub fn with_setting(mut self, key: impl Into, value: impl Into) -> Self { + self.settings.insert(key.into(), value.into()); + self + } + + /// The same table keeping or stripping the reader's query. + #[must_use] + pub fn with_origin_query(mut self, policy: Option) -> Self { + self.origin_query = policy; + self + } + + pub(crate) fn normalize(&mut self) { + self.signer = self.signer.trim().to_owned(); + } + + /// Refuses a table that names no signer, or names one in a form no + /// crate could offer, naming `setting`. + pub(crate) fn check(&self, setting: &str) -> Result<(), Report> { + if self.signer.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("[{setting}] names no signer. Write `type`"), + })); + } + if !crate::module_name::is_valid(&self.signer) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{setting}] type `{}` is not a signer's name. A signer is named in lower \ + case letters, digits, `_` or `-`", + self.signer + ), + })); + } + Ok(()) + } +} + +/// Signs a request to an origin. +/// +/// The address and the headers are final when a signer is called, the `Host` +/// header among them, so the signature covers exactly what is sent. +pub trait OriginSigner: Send + Sync { + /// Signs the request `headers` carry for `method` on `url` at `now`, + /// replacing any earlier signature. + /// + /// # Errors + /// + /// When a header the signature needs is missing or cannot be written. + fn sign( + &self, + method: &Method, + url: &Url, + headers: &mut HeaderMap, + now: SystemTime, + ) -> Result<(), Report>; + + /// Removes every header in the scheme's own namespace, so a header the + /// reader chose cannot reach the origin beside the signature. + fn remove_namespace_headers(&self, headers: &mut HeaderMap); +} + +/// A setting of a signer's table that holds the name of a secret. +#[derive(Debug, Clone, Copy)] +pub struct SignerSecret { + /// The setting's key in the `auth` table. + pub name: &'static str, + /// The key name read when the setting is left out, or none for a + /// setting that is only looked up when written. + pub default: Option<&'static str>, +} + +/// Builds the signer `settings` configure, being the `auth` table without +/// the keys core reads, where `setting` names the table in a refusal. +pub type OriginSignerBuildFn = + fn(&Map, &str) -> Result, Report>; + +/// A signer a crate offers, which an `auth` table selects with +/// `type = ""`. +pub struct OriginSignerModule { + /// The name an `auth` table writes. + pub name: &'static str, + /// The settings of the signer's table that hold the name of a secret. + pub secrets: &'static [SignerSecret], + /// Builds the signer from its table. + pub build: OriginSignerBuildFn, +} + +impl fmt::Debug for OriginSignerModule { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OriginSignerModule") + .field("name", &self.name) + .field("secrets", &self.secrets) + .finish_non_exhaustive() + } +} + +/// The signing a location applies to each request: the signer and whether +/// the reader's query is kept. +#[derive(Clone)] +pub struct OriginSigning { + signer: Arc, + origin_query: Option, +} + +impl fmt::Debug for OriginSigning { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OriginSigning") + .field("origin_query", &self.origin_query) + .finish_non_exhaustive() + } +} + +impl OriginSigning { + /// Signing through `signer`, keeping or stripping the reader's query as + /// `origin_query` says. + #[must_use] + pub fn new(signer: Arc, origin_query: Option) -> Self { + Self { + signer, + origin_query, + } + } + + /// Signs the request `headers` carry for `method` on `url`, now. + /// + /// # Errors + /// + /// As [`OriginSigner::sign`]. + pub fn sign( + &self, + method: &Method, + url: &Url, + headers: &mut HeaderMap, + ) -> Result<(), Report> { + self.signer + .sign(method, url, headers, crate::ec::std_system_time_now()) + } + + /// Removes every header in the signer's namespace, see + /// [`OriginSigner::remove_namespace_headers`]. + pub fn remove_namespace_headers(&self, headers: &mut HeaderMap) { + self.signer.remove_namespace_headers(headers); + } + + /// Whether the reader's query is kept on the signed request, when the + /// `auth` table says. + #[must_use] + pub fn origin_query_policy(&self) -> Option { + self.origin_query + } +} + +/// The signers a deployment's builders offer, by name. +#[derive(Clone, Default)] +pub struct OriginSigners { + offered: Vec<&'static OriginSignerModule>, +} + +impl fmt::Debug for OriginSigners { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OriginSigners") + .field("offered", &self.names()) + .finish() + } +} + +impl OriginSigners { + /// The signers `offered`, in that order. + #[must_use] + pub fn new(offered: Vec<&'static OriginSignerModule>) -> Self { + Self { offered } + } + + /// The signers `builders` offer, or a refusal when two of them offer one + /// name. + /// + /// # Errors + /// + /// Naming the signer and both builders. + pub fn offered_by(builders: &[IntegrationBuilder]) -> Result> { + let mut offered: Vec<(&'static OriginSignerModule, &'static str)> = Vec::new(); + for builder in builders { + let Some(module) = builder.origin_signer() else { + continue; + }; + if let Some((_, by)) = offered.iter().find(|(known, _)| known.name == module.name) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "`{}` and `{by}` both offer the origin signer `{}`", + builder.id(), + module.name + ), + })); + } + offered.push((module, builder.id())); + } + Ok(Self::new( + offered.into_iter().map(|(module, _)| module).collect(), + )) + } + + /// The names on offer, in the order offered. + #[must_use] + pub fn names(&self) -> Vec<&'static str> { + self.offered.iter().map(|module| module.name).collect() + } + + /// The signer offered under `name`, if any. + #[must_use] + pub fn get(&self, name: &str) -> Option<&'static OriginSignerModule> { + self.offered + .iter() + .copied() + .find(|module| module.name == name) + } + + /// The signing `auth` selects, built from its table, or a refusal naming + /// `setting`. + /// + /// # Errors + /// + /// When `auth` names a signer nothing offers, with the names on offer, or + /// when the signer cannot use its table. + pub fn build( + &self, + auth: &OriginAuth, + setting: &str, + ) -> Result> { + let Some(module) = self.get(&auth.signer) else { + let names = self.names(); + let offer = if names.is_empty() { + "It offers none".to_owned() + } else { + format!("The origin signers it offers are [{}]", names.join(", ")) + }; + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{setting}] type `{}` names an origin signer this deployment does not \ + offer. {offer}", + auth.signer + ), + })); + }; + let signer = (module.build)(&auth.settings, setting)?; + Ok(OriginSigning::new(signer, auth.origin_query)) + } +} + +/// The `auth` table at `source.origin.auth` of `node`, a serialized +/// configuration or one of its asset routes. +fn auth_table_of(node: &mut Value) -> Option<&mut Map> { + node.get_mut("source")? + .get_mut("origin")? + .get_mut("auth")? + .as_object_mut() +} + +/// Calls `visit` with every `auth` table a serialized configuration holds +/// and the path a refusal names: the publisher's source's, then each asset +/// route's. +fn each_auth_table(data: &mut Value, mut visit: impl FnMut(&str, &mut Map)) { + if let Some(table) = auth_table_of(data) { + visit("source.origin.auth", table); + } + let Some(routes) = data + .get_mut("proxy") + .and_then(|proxy| proxy.get_mut("asset_routes")) + .and_then(Value::as_array_mut) + else { + return; + }; + for (index, route) in routes.iter_mut().enumerate() { + if let Some(table) = auth_table_of(route) { + visit( + &format!("proxy.asset_routes[{index}].source.origin.auth"), + table, + ); + } + } +} + +/// The signer `table` names, when `signers` offers it. +fn signer_of( + table: &Map, + signers: &OriginSigners, +) -> Option<&'static OriginSignerModule> { + table + .get("type") + .and_then(Value::as_str) + .and_then(|name| signers.get(name.trim())) +} + +/// Writes the default key name of every secret setting an `auth` table +/// leaves out, in a serialized configuration whose secrets have not been +/// looked up yet, so the lookup finds the setting as a written one. +pub(crate) fn fill_secret_defaults(data: &mut Value, signers: &OriginSigners) { + each_auth_table(data, |_, table| { + let Some(module) = signer_of(table, signers) else { + return; + }; + for secret in module.secrets { + if let Some(default) = secret.default + && !table.contains_key(secret.name) + { + table.insert(secret.name.to_owned(), Value::String(default.to_owned())); + } + } + }); +} + +/// The secret leaves the offered signers declare, at both places an `auth` +/// table can be written. Every level is optional, so a source that signs +/// with nothing, or leaves a setting out, is passed over. +pub(crate) fn secret_fields(signers: &OriginSigners) -> Vec { + let optional = |name: &'static str| SecretPathSegment::OptionalField(Cow::Borrowed(name)); + let mut fields = Vec::new(); + for module in &signers.offered { + for secret in module.secrets { + let leaf = SecretPathSegment::Field(Cow::Borrowed(secret.name)); + fields.push(SecretField { + kind: SecretKind::KeyInDefault, + optional: true, + path: vec![ + optional("source"), + optional("origin"), + optional("auth"), + leaf.clone(), + ], + }); + fields.push(SecretField { + kind: SecretKind::KeyInDefault, + optional: true, + path: vec![ + optional("proxy"), + optional("asset_routes"), + SecretPathSegment::ArrayEach, + optional("source"), + optional("origin"), + optional("auth"), + leaf, + ], + }); + } + } + fields +} + +/// Refuses an `auth` table whose written secret setting names no key, for +/// a check that runs while the settings still hold key names. +/// +/// # Errors +/// +/// Naming the setting's path. +pub(crate) fn check_key_references( + settings: &Settings, + signers: &OriginSigners, +) -> Result<(), Report> { + let mut data = serde_json::to_value(settings).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("the settings could not be serialized: {error}"), + }) + })?; + let mut blank = None; + each_auth_table(&mut data, |path, table| { + if blank.is_some() { + return; + } + let Some(module) = signer_of(table, signers) else { + return; + }; + for secret in module.secrets { + if let Some(value) = table.get(secret.name) + && value.as_str().is_none_or(|key| key.trim().is_empty()) + { + blank = Some(format!("{path}.{}", secret.name)); + return; + } + } + }); + match blank { + Some(path) => Err(Report::new(TrustedServerError::Configuration { + message: format!("secret key reference at `{path}` must not be empty"), + })), + None => Ok(()), + } +} + +/// A signer for core's tests, which cannot link the crates that supply the +/// real ones. +/// +/// Its table is `realm`, which it refuses empty, `key_name`, a secret with +/// a default key name, and `token_name`, a secret with none. It writes +/// `Authorization: Fixture `, so a test can +/// check exactly what was signed, and `x-fixture-signed`, and its namespace +/// is every `x-fixture-*` header. +#[cfg(test)] +pub(crate) mod fixture { + use std::sync::Arc; + use std::time::SystemTime; + + use error_stack::Report; + use http::{HeaderMap, HeaderValue, Method, header}; + use serde::Deserialize; + use serde_json::{Map, Value}; + use url::Url; + + use super::{OriginAuth, OriginSigner, OriginSignerModule, OriginSigners, SignerSecret}; + use crate::error::TrustedServerError; + use crate::integrations::IntegrationBuilder; + + /// The name an `auth` table writes to select the fixture. + pub(crate) const SIGNER: &str = "fixture_scheme"; + + /// The key name `key_name` takes when left out. + pub(crate) const DEFAULT_KEY_NAME: &str = "fixture_key"; + + /// The id of the builder that offers the fixture. + pub(crate) const INTEGRATION_ID: &str = "fixture-signer"; + + pub(crate) const SECRETS: &[SignerSecret] = &[ + SignerSecret { + name: "key_name", + default: Some(DEFAULT_KEY_NAME), + }, + SignerSecret { + name: "token_name", + default: None, + }, + ]; + + pub(crate) static MODULE: OriginSignerModule = OriginSignerModule { + name: SIGNER, + secrets: SECRETS, + build, + }; + + /// A builder offering the fixture, for a registry or a settings load. + pub(crate) fn builder() -> IntegrationBuilder { + IntegrationBuilder::implementations(INTEGRATION_ID, "trusted-server-core") + .with_origin_signer(&MODULE) + } + + /// The fixture, offered on its own. + pub(crate) fn signers() -> OriginSigners { + OriginSigners::new(vec![&MODULE]) + } + + /// An `auth` table selecting the fixture in `realm`, with the default + /// key name. + pub(crate) fn auth(realm: &str) -> OriginAuth { + OriginAuth::new(SIGNER).with_setting("realm", realm) + } + + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + realm: String, + #[serde(default = "default_key_name")] + key_name: String, + #[serde(default)] + token_name: Option, + } + + fn default_key_name() -> String { + DEFAULT_KEY_NAME.to_owned() + } + + #[derive(Debug)] + pub(crate) struct FixtureSigner { + realm: String, + key: String, + token: Option, + } + + fn build( + settings: &Map, + setting: &str, + ) -> Result, Report> { + let read: FixtureSettings = serde_json::from_value(Value::Object(settings.clone())) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("[{setting}] could not be read by {SIGNER}: {error}"), + }) + })?; + if read.realm.trim().is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("[{setting}] {SIGNER} realm must not be empty"), + })); + } + Ok(Arc::new(FixtureSigner { + realm: read.realm, + key: read.key_name, + token: read.token_name, + })) + } + + impl OriginSigner for FixtureSigner { + fn sign( + &self, + method: &Method, + url: &Url, + headers: &mut HeaderMap, + _now: SystemTime, + ) -> Result<(), Report> { + let host = headers + .get(header::HOST) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| { + Report::new(TrustedServerError::Proxy { + message: "missing Host header to sign".to_owned(), + }) + })? + .to_owned(); + let value = expected_authorization(&self.key, method, url.as_str(), &host); + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_str(&value).map_err(|error| { + Report::new(TrustedServerError::InvalidHeaderValue { + message: format!("fixture signature: {error}"), + }) + })?, + ); + headers.insert("x-fixture-signed", HeaderValue::from_static("1")); + headers.insert( + "x-fixture-realm", + HeaderValue::from_str(&self.realm).map_err(|error| { + Report::new(TrustedServerError::InvalidHeaderValue { + message: format!("fixture realm: {error}"), + }) + })?, + ); + if let Some(token) = &self.token { + headers.insert( + "x-fixture-token", + HeaderValue::from_str(token).map_err(|error| { + Report::new(TrustedServerError::InvalidHeaderValue { + message: format!("fixture token: {error}"), + }) + })?, + ); + } + Ok(()) + } + + fn remove_namespace_headers(&self, headers: &mut HeaderMap) { + let names: Vec<_> = headers + .keys() + .filter(|name| name.as_str().starts_with("x-fixture-")) + .cloned() + .collect(); + for name in names { + headers.remove(&name); + } + } + } + + /// The `Authorization` the fixture writes with `key` for `method` on + /// `url` sent with `host`. + pub(crate) fn expected_authorization( + key: &str, + method: &Method, + url: &str, + host: &str, + ) -> String { + format!("Fixture {key} {method} {url} {host}") + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::integrations::IntegrationBuilder; + + fn table(value: Value) -> Map { + match value { + Value::Object(table) => table, + _ => unreachable!("the tests write objects"), + } + } + + #[test] + fn an_auth_table_reads_its_type_and_query_policy_and_keeps_the_rest() { + let auth: OriginAuth = serde_json::from_value(json!({ + "type": "fixture_scheme", + "origin_query": "strip", + "realm": "site", + "key_name": "origin_key", + })) + .expect("should read the table"); + + assert_eq!(auth.signer, "fixture_scheme"); + assert_eq!(auth.origin_query, Some(OriginQueryPolicy::Strip)); + assert_eq!( + auth.settings, + table(json!({ "realm": "site", "key_name": "origin_key" })), + "the signer's settings should be kept apart from core's keys" + ); + assert_eq!( + serde_json::to_value(&auth).expect("should serialize"), + json!({ + "type": "fixture_scheme", + "origin_query": "strip", + "realm": "site", + "key_name": "origin_key", + }), + "the table should serialize as written" + ); + } + + #[test] + fn an_auth_table_debug_shows_setting_names_and_no_values() { + let auth = fixture::auth("site").with_setting("key_name", "canary-key-value"); + + let debug = format!("{auth:?}"); + + assert!(debug.contains("fixture_scheme") && debug.contains("key_name")); + assert!( + !debug.contains("canary-key-value"), + "a setting's value may be a secret: {debug}" + ); + } + + #[test] + fn an_auth_table_must_name_a_signer_in_a_form_a_crate_could_offer() { + for (signer, expected) in [ + ("", "names no signer"), + ("Not A Name", "is not a signer's name"), + ] { + let err = OriginAuth::new(signer) + .check("source.origin.auth") + .expect_err("should refuse"); + assert!( + format!("{err:?}").contains(expected), + "`{signer}` should be refused saying `{expected}`: {err:?}" + ); + } + OriginAuth::new("fixture_scheme") + .check("source.origin.auth") + .expect("a valid name passes"); + } + + #[test] + fn a_type_nothing_offers_is_refused_with_the_names_on_offer() { + let err = fixture::signers() + .build(&OriginAuth::new("other_scheme"), "source.origin.auth") + .expect_err("should refuse a type nothing offers"); + let message = format!("{err:?}"); + assert!( + message.contains("[source.origin.auth] type `other_scheme` names an origin signer this deployment does not offer") + && message.contains("[fixture_scheme]"), + "should name the setting and the signers on offer: {message}" + ); + + let err = OriginSigners::default() + .build(&OriginAuth::new("other_scheme"), "source.origin.auth") + .expect_err("should refuse with nothing on offer"); + assert!( + format!("{err:?}").contains("It offers none"), + "should say nothing is on offer: {err:?}" + ); + } + + #[test] + fn a_table_the_signer_cannot_use_is_refused_naming_the_setting() { + for (auth, expected) in [ + ( + fixture::auth(""), + "[source.origin.auth] fixture_scheme realm must not be empty", + ), + ( + fixture::auth("site").with_setting("bucket", "x"), + "unknown field `bucket`", + ), + (OriginAuth::new(fixture::SIGNER), "missing field `realm`"), + ] { + let err = fixture::signers() + .build(&auth, "source.origin.auth") + .expect_err("should refuse the table"); + assert!( + format!("{err:?}").contains(expected), + "should say `{expected}`: {err:?}" + ); + } + } + + #[test] + fn a_built_signing_signs_the_final_address_and_host() { + let signing = fixture::signers() + .build( + &fixture::auth("site").with_origin_query(Some(OriginQueryPolicy::Strip)), + "source.origin.auth", + ) + .expect("should build"); + let url = Url::parse("https://storage.example.com/bucket/site/page").expect("url"); + let mut headers = HeaderMap::new(); + headers.insert( + http::header::HOST, + http::HeaderValue::from_static("storage.example.com"), + ); + headers.insert("x-fixture-reader", http::HeaderValue::from_static("chosen")); + + signing.remove_namespace_headers(&mut headers); + signing + .sign(&Method::GET, &url, &mut headers) + .expect("should sign"); + + assert_eq!( + signing.origin_query_policy(), + Some(OriginQueryPolicy::Strip) + ); + assert_eq!( + headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some( + fixture::expected_authorization( + fixture::DEFAULT_KEY_NAME, + &Method::GET, + url.as_str(), + "storage.example.com", + ) + .as_str() + ) + ); + assert!( + headers.get("x-fixture-reader").is_none(), + "a reader's header in the namespace should be removed" + ); + } + + #[test] + fn two_builders_offering_one_signer_are_refused() { + let err = OriginSigners::offered_by(&[ + fixture::builder(), + IntegrationBuilder::implementations("second", "elsewhere") + .with_origin_signer(&fixture::MODULE), + ]) + .expect_err("should refuse the second offer"); + assert!( + format!("{err:?}").contains( + "`second` and `fixture-signer` both offer the origin signer `fixture_scheme`" + ), + "{err:?}" + ); + + let offered = OriginSigners::offered_by(&[fixture::builder()]).expect("one offer"); + assert_eq!(offered.names(), vec![fixture::SIGNER]); + } + + #[test] + fn secret_defaults_are_written_into_each_auth_table_naming_the_signer() { + let mut data = json!({ + "source": { "module": "origin", "origin": { + "url": "https://storage.example.com", + "auth": { "type": "fixture_scheme", "realm": "site" }, + }}, + "proxy": { "asset_routes": [ + { "prefix": "/a/", "source": { "module": "origin", "origin": { + "url": "https://a.example.com", + "auth": { "type": "fixture_scheme", "realm": "a", "key_name": "written" }, + }}}, + { "prefix": "/b/", "source": { "module": "origin", "origin": { + "url": "https://b.example.com", + "auth": { "type": "other_scheme", "realm": "b" }, + }}}, + { "prefix": "/c/", "source": { "module": "origin", "origin": { + "url": "https://c.example.com", + }}}, + ]}, + }); + + fill_secret_defaults(&mut data, &fixture::signers()); + + assert_eq!( + data["source"]["origin"]["auth"]["key_name"], + json!(fixture::DEFAULT_KEY_NAME), + "a left out setting with a default should take it" + ); + assert!( + data["source"]["origin"]["auth"].get("token_name").is_none(), + "a setting with no default should stay left out" + ); + assert_eq!( + data["proxy"]["asset_routes"][0]["source"]["origin"]["auth"]["key_name"], + json!("written"), + "a written setting should be kept" + ); + assert!( + data["proxy"]["asset_routes"][1]["source"]["origin"]["auth"] + .get("key_name") + .is_none(), + "a table naming another signer should be left alone" + ); + } + + #[test] + fn secret_fields_are_declared_at_both_auth_paths() { + let declared: Vec = secret_fields(&fixture::signers()) + .iter() + .map(SecretField::dotted_path) + .collect(); + + assert_eq!( + declared, + vec![ + "source.origin.auth.key_name", + "proxy.asset_routes[*].source.origin.auth.key_name", + "source.origin.auth.token_name", + "proxy.asset_routes[*].source.origin.auth.token_name", + ] + ); + } + + #[test] + fn a_written_secret_setting_naming_no_key_is_refused_naming_its_path() { + use crate::test_support::tests::{create_test_settings, edit_origin, set_route_auth}; + + let mut settings = create_test_settings(); + edit_origin(&mut settings, |origin| { + origin.url = "https://storage.example.com/site".to_owned(); + origin.auth = Some(fixture::auth("site").with_setting("key_name", " ")); + }); + let err = check_key_references(&settings, &fixture::signers()) + .expect_err("should refuse a blank key name"); + assert!( + format!("{err:?}").contains( + "secret key reference at `source.origin.auth.key_name` must not be empty" + ), + "{err:?}" + ); + + let mut settings = create_test_settings(); + let mut route = + crate::settings::ProxyAssetRoute::new("/assets/", "https://assets.example.com"); + set_route_auth( + &mut route, + Some(fixture::auth("assets").with_setting("token_name", "")), + ); + settings.proxy.asset_routes.push(route); + let err = check_key_references(&settings, &fixture::signers()) + .expect_err("should refuse a blank token name"); + assert!( + format!("{err:?}").contains( + "secret key reference at `proxy.asset_routes[0].source.origin.auth.token_name` must not be empty" + ), + "{err:?}" + ); + + let mut settings = create_test_settings(); + edit_origin(&mut settings, |origin| { + origin.url = "https://storage.example.com/site".to_owned(); + origin.auth = Some(fixture::auth("site")); + }); + check_key_references(&settings, &fixture::signers()) + .expect("a left out setting names its default key"); + } +} diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 223dde0c33..9f577c6b82 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -27,8 +27,7 @@ use crate::platform::{ DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, PlatformHttpRequest, PlatformResponse, RuntimeServices, }; -use crate::s3_sigv4::{self, S3Credentials}; -use crate::settings::{AssetOriginAuth, OriginQueryPolicy, ProxyAssetRoute, Settings}; +use crate::settings::{OriginQueryPolicy, ProxyAssetRoute, Settings}; use crate::source::{Source, SourceLocation, SourceRequest}; use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, StreamingPipeline}; @@ -919,48 +918,6 @@ fn asset_origin_host_header( }) } -/// Signs an origin request whose address and headers are otherwise final. -/// -/// The one signing path for origin auth, used by asset routes and by the -/// publisher origin alike. -pub(crate) fn apply_asset_origin_auth( - method: &Method, - target_url: &url::Url, - headers: &mut http::HeaderMap, - auth: &AssetOriginAuth, -) -> Result<(), Report> { - match auth { - AssetOriginAuth::S3SigV4(config) => { - let credentials = S3Credentials { - access_key_id: config.access_key_id.expose().clone(), - secret_access_key: config.secret_access_key.clone(), - session_token: config.session_token.clone(), - }; - s3_sigv4::sign_headers( - method, - target_url, - headers, - &config.region, - &credentials, - // s3_sigv4 converts this via chrono's `DateTime::::from`, which - // only accepts `std::time::SystemTime`. - crate::ec::std_system_time_now(), - ) - } - } -} - -/// Removes every header in `auth`'s signing namespace, so a header a reader -/// sent there cannot reach the origin beside the signature. -pub(crate) fn remove_origin_auth_namespace_headers( - headers: &mut http::HeaderMap, - auth: &AssetOriginAuth, -) { - match auth { - AssetOriginAuth::S3SigV4(_) => s3_sigv4::remove_namespace_headers(headers), - } -} - fn build_asset_platform_request( method: &Method, target_url: &url::Url, @@ -1031,17 +988,21 @@ fn apply_no_store_cache_control(response: &mut Response) { apply_no_store_private_to_headers(response.headers_mut()); } -fn should_preflight_s3( +fn should_preflight_signed_origin( location: &SourceLocation, image_optimizer_enabled: bool, request_method: &Method, ) -> bool { image_optimizer_enabled - && matches!(location.auth(), Some(AssetOriginAuth::S3SigV4(_))) + && location.auth().is_some() && (request_method == Method::GET || request_method == Method::HEAD) } -async fn preflight_s3_origin_for_image_optimizer( +/// Asks a signed origin for the object with a `HEAD` before the image +/// optimizer is asked for it, because the optimizer can mask or transform +/// the origin's own error. A missing or refused object is answered with that +/// error and the optimizer is left out of the failure path. +async fn preflight_signed_origin_for_image_optimizer( services: &RuntimeServices, location: &SourceLocation, target_url: &url::Url, @@ -1049,15 +1010,12 @@ async fn preflight_s3_origin_for_image_optimizer( unsigned_headers: &http::HeaderMap, backend_name: &str, ) -> Result, Report> { - let Some(auth @ AssetOriginAuth::S3SigV4(_)) = location.auth() else { + let Some(auth) = location.auth() else { return Ok(None); }; - // Fastly Image Optimizer can mask or transform S3 origin errors. A signed - // HEAD preflight lets missing or unauthorized objects return raw S3 errors - // without invoking IO on the failure path. let mut head_headers = unsigned_headers.clone(); - apply_asset_origin_auth(&Method::HEAD, target_url, &mut head_headers, auth)?; + auth.sign(&Method::HEAD, target_url, &mut head_headers)?; let head_response = send_asset_origin_request( services, backend_name, @@ -1080,7 +1038,7 @@ async fn preflight_s3_origin_for_image_optimizer( } let mut get_headers = unsigned_headers.clone(); - apply_asset_origin_auth(&Method::GET, target_url, &mut get_headers, auth)?; + auth.sign(&Method::GET, target_url, &mut get_headers)?; let mut response = send_asset_origin_request( services, backend_name, @@ -1103,15 +1061,15 @@ async fn preflight_s3_origin_for_image_optimizer( /// is path rewrite, profile-table Image Optimizer metadata extraction, optional /// origin query stripping, optional origin authentication, then platform send. /// -/// The origin query policy is applied before S3 signing so the signature covers +/// The origin query policy is applied before signing so the signature covers /// the exact URL sent to the asset origin. Image Optimizer metadata remains /// separate from the origin URL and is translated by the platform adapter. /// /// # Errors /// /// Returns an error if the configured origin URL is invalid, backend -/// registration fails, S3 credentials cannot be read, signing fails, image -/// optimizer metadata cannot be built, or the upstream request cannot be sent. +/// registration fails, signing fails, image optimizer metadata cannot be +/// built, or the upstream request cannot be sent. pub async fn handle_asset_proxy_request( settings: &Settings, services: &RuntimeServices, @@ -1188,8 +1146,8 @@ pub async fn handle_asset_proxy_request( }; outbound_headers.insert(header::HOST, host_header); - if should_preflight_s3(&location, image_optimizer.is_some(), req.method()) - && let Some(response) = preflight_s3_origin_for_image_optimizer( + if should_preflight_signed_origin(&location, image_optimizer.is_some(), req.method()) + && let Some(response) = preflight_signed_origin_for_image_optimizer( services, &location, &target_url, @@ -1203,7 +1161,8 @@ pub async fn handle_asset_proxy_request( } if let Some(auth) = location.auth() { - apply_asset_origin_auth(req.method(), &target_url, &mut outbound_headers, auth)?; + auth.remove_namespace_headers(&mut outbound_headers); + auth.sign(req.method(), &target_url, &mut outbound_headers)?; } let stream_response = services.http_client().supports_streaming_responses(); @@ -2400,6 +2359,7 @@ mod tests { use crate::constants::{HEADER_ACCEPT, HEADER_X_FORWARDED_FOR}; use crate::creative; use crate::error::{IntoHttpResponse, TrustedServerError}; + use crate::origin_auth::fixture; use crate::platform::RuntimeServices; use crate::platform::test_support::{ HashMapSecretStore, StubHttpClient, build_services_with_http_client, @@ -2409,11 +2369,10 @@ mod tests { PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, StoreName, }; - use crate::redacted::Redacted; use crate::settings::{ - AssetImageOptimizerConfig, AssetOriginAuth, ImageOptimizerAspectRatioConfig, + AssetImageOptimizerConfig, ImageOptimizerAspectRatioConfig, ImageOptimizerCropOffsetsConfig, ImageOptimizerProfileSet, ImageOptimizerSettings, - OriginQueryPolicy, ProxyAssetRoute, S3SigV4AuthConfig, Settings, UnknownProfilePolicy, + OriginQueryPolicy, ProxyAssetRoute, Settings, UnknownProfilePolicy, }; use crate::test_support::tests::{ crate_test_settings_str, create_test_settings, set_route_auth, @@ -5028,16 +4987,19 @@ mod tests { route .origin_source() .expect("the route should select the origin source") - .location() + .location(&fixture::signers(), "proxy.asset_routes source.origin") .expect("the origin source should give a location") } - /// `route`'s `origin` source, as the registry builds it. + /// `route`'s `origin` source, as the registry builds it with the fixture + /// signer on offer. fn origin_source_of(route: &ProxyAssetRoute) -> crate::source::OriginSource { crate::source::OriginSource::new( &route .origin_source() .expect("the route should select the origin source"), + &fixture::signers(), + "proxy.asset_routes source.origin", ) .expect("should build the route's source") } @@ -5914,36 +5876,30 @@ mod tests { } } - fn test_s3_secrets() -> HashMap> { - HashMap::from([ - ( - "access_key_id".to_string(), - b"AKIAIOSFODNN7EXAMPLE".to_vec(), - ), - ( - "secret_access_key".to_string(), - b"wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_vec(), - ), - ]) + /// The resolved key the signed route tests sign with. + const SIGNED_ROUTE_KEY: &str = "resolved-route-key"; + + /// The address of the signed route tests' origin. + const SIGNED_ROUTE_ORIGIN: &str = "https://bucket.storage.example.com"; + + fn test_signed_route_secrets() -> HashMap> { + HashMap::from([( + "fixture_key".to_string(), + SIGNED_ROUTE_KEY.as_bytes().to_vec(), + )]) } - fn test_s3_image_optimizer_route() -> ProxyAssetRoute { - let mut route = ProxyAssetRoute::new( - "/.images/", - "https://examplebucket.s3.us-east-1.amazonaws.com", - ); + /// The `Authorization` the fixture signer writes for `method` on `uri` + /// sent to the signed route tests' origin. + fn signed_route_authorization(method: &Method, uri: &str) -> String { + fixture::expected_authorization(SIGNED_ROUTE_KEY, method, uri, "bucket.storage.example.com") + } + + fn test_signed_image_optimizer_route() -> ProxyAssetRoute { + let mut route = ProxyAssetRoute::new("/.images/", SIGNED_ROUTE_ORIGIN); set_route_auth( &mut route, - Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "us-east-1".to_string(), - secret_store: None, - access_key_id: Redacted::new("AKIAIOSFODNN7EXAMPLE".to_string()), - secret_access_key: Redacted::new( - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), - ), - session_token: None, - origin_query: None, - })), + Some(fixture::auth("images").with_setting("key_name", SIGNED_ROUTE_KEY)), ); route.image_optimizer = Some(AssetImageOptimizerConfig { enabled: true, @@ -5955,44 +5911,29 @@ mod tests { } #[test] - fn handle_asset_proxy_request_signs_s3_and_strips_transform_query() { + fn handle_asset_proxy_request_signs_the_final_address_and_strips_transform_query() { futures::executor::block_on(async { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"ok".to_vec()); - let mut secrets = HashMap::new(); - secrets.insert( - "access_key_id".to_string(), - b"AKIAIOSFODNN7EXAMPLE".to_vec(), - ); - secrets.insert( - "secret_access_key".to_string(), - b"wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_vec(), - ); let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(secrets), + HashMapSecretStore::new(test_signed_route_secrets()), Arc::clone(&stub) as Arc, ); let settings = create_test_settings(); - let req = build_http_request( + let mut req = build_http_request( Method::GET, "https://www.example.com/.images/foo.jpg?profile=medium&ar=1-1", ); - let mut route = ProxyAssetRoute::new( - "/.images/", - "https://examplebucket.s3.us-east-1.amazonaws.com", - ); + req.headers_mut() + .insert("x-fixture-reader", HeaderValue::from_static("chosen")); + let mut route = ProxyAssetRoute::new("/.images/", SIGNED_ROUTE_ORIGIN); set_route_auth( &mut route, - Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "us-east-1".to_string(), - secret_store: None, - access_key_id: Redacted::new("AKIAIOSFODNN7EXAMPLE".to_string()), - secret_access_key: Redacted::new( - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), - ), - session_token: None, - origin_query: Some(OriginQueryPolicy::Strip), - })), + Some( + fixture::auth("images") + .with_setting("key_name", SIGNED_ROUTE_KEY) + .with_origin_query(Some(OriginQueryPolicy::Strip)), + ), ); handle_asset_proxy_request( @@ -6003,13 +5944,13 @@ mod tests { &origin_source_of(&route), ) .await - .expect("should proxy signed S3 asset request"); + .expect("should proxy the signed asset request"); - let uris = stub.recorded_request_uris(); + let sent_uri = "https://bucket.storage.example.com/.images/foo.jpg"; assert_eq!( - uris, - vec!["https://examplebucket.s3.us-east-1.amazonaws.com/.images/foo.jpg"], - "should strip transform query before signing and sending to S3" + stub.recorded_request_uris(), + vec![sent_uri], + "should strip the transform query before signing and sending" ); let headers = stub.recorded_request_headers(); let sent = &headers[0]; @@ -6018,25 +5959,24 @@ mod tests { }; assert_eq!( header_value("host").as_deref(), - Some("examplebucket.s3.us-east-1.amazonaws.com"), - "should sign for the S3 origin host" + Some("bucket.storage.example.com"), + "should sign for the origin host" ); - assert!( - header_value("authorization") - .as_deref() - .is_some_and(|value| value.starts_with("AWS4-HMAC-SHA256 Credential=")), - "should add a SigV4 Authorization header" + assert_eq!( + header_value("authorization").as_deref(), + Some(signed_route_authorization(&Method::GET, sent_uri).as_str()), + "should sign exactly the address and host that were sent" ); assert_eq!( - header_value("x-amz-content-sha256").as_deref(), - Some("UNSIGNED-PAYLOAD"), - "should use unsigned payload for read-only asset requests" + header_value("x-fixture-reader"), + None, + "a reader's header in the signer's namespace should not be sent" ); }); } #[test] - fn handle_asset_proxy_request_uses_resolved_s3_credentials_without_store_reads() { + fn handle_asset_proxy_request_signs_with_the_resolved_settings_without_store_reads() { futures::executor::block_on(async { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, Vec::new()); @@ -6055,19 +5995,14 @@ mod tests { Method::GET, "https://www.example.com/.images/foo.jpg?profile=medium", ); - let mut route = test_s3_image_optimizer_route(); + let mut route = test_signed_image_optimizer_route(); set_route_auth( &mut route, - Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "us-east-1".to_string(), - secret_store: None, - access_key_id: Redacted::new("AKIAIOSFODNN7EXAMPLE".to_string()), - secret_access_key: Redacted::new( - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), - ), - session_token: Some(Redacted::new("session-token".to_string())), - origin_query: None, - })), + Some( + fixture::auth("images") + .with_setting("key_name", SIGNED_ROUTE_KEY) + .with_setting("token_name", "resolved-token"), + ), ); handle_asset_proxy_request( @@ -6078,27 +6013,24 @@ mod tests { &origin_source_of(&route), ) .await - .expect("should proxy optimized S3 asset request"); + .expect("should proxy the optimized signed asset request"); assert_eq!( stub.recorded_request_methods(), vec!["HEAD", "GET"], - "should sign both the S3 preflight and final request" + "should sign both the preflight and the final request" ); assert_eq!( - observed_secret_store.read_count("AKIAIOSFODNN7EXAMPLE"), + observed_secret_store.read_count("fixture_key"), 0, - "should not read S3 credentials from the runtime secret store" + "should not read the signer's settings from the runtime secret store" ); let headers = stub.recorded_request_headers(); assert!( - headers + headers.iter().all(|sent| sent .iter() - .all(|sent| sent - .iter() - .any(|(name, value)| name == "x-amz-security-token" - && value == "session-token")), - "should sign both requests with the cached session token" + .any(|(name, value)| name == "x-fixture-token" && value == "resolved-token")), + "should sign both requests with the resolved settings" ); }); } @@ -6315,12 +6247,12 @@ mod tests { } #[test] - fn handle_asset_proxy_request_skips_s3_preflight_for_svg_image_optimizer_routes() { + fn handle_asset_proxy_request_skips_the_signed_preflight_for_svg_image_optimizer_routes() { futures::executor::block_on(async { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"raw-svg".to_vec()); let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(test_s3_secrets()), + HashMapSecretStore::new(test_signed_route_secrets()), Arc::clone(&stub) as Arc, ); let settings = create_test_settings(); @@ -6328,7 +6260,7 @@ mod tests { Method::GET, "https://www.example.com/.images/logo.svg?profile=medium", ); - let route = test_s3_image_optimizer_route(); + let route = test_signed_image_optimizer_route(); let response = handle_asset_proxy_request( &settings, @@ -6338,7 +6270,7 @@ mod tests { &origin_source_of(&route), ) .await - .expect("should proxy raw SVG asset through S3 route") + .expect("should proxy raw SVG asset through the signed route") .into_response() .expect("should return buffered asset response"); @@ -6346,11 +6278,11 @@ mod tests { assert_eq!( stub.recorded_request_methods(), vec!["GET"], - "SVG IO bypass should not add an S3 preflight" + "SVG IO bypass should not add a signed preflight" ); assert_eq!( stub.recorded_request_uris(), - vec!["https://examplebucket.s3.us-east-1.amazonaws.com/.images/logo.svg"], + vec!["https://bucket.storage.example.com/.images/logo.svg"], "SVG IO bypass should still strip the transform query" ); assert_eq!( @@ -6362,14 +6294,14 @@ mod tests { } #[test] - fn handle_asset_proxy_request_preflights_s3_before_image_optimizer() { + fn handle_asset_proxy_request_preflights_a_signed_origin_before_image_optimizer() { futures::executor::block_on(async { let stub = Arc::new(StubHttpClient::new()); stub.set_streaming_responses_supported(true); stub.push_response(200, Vec::new()); stub.push_response(200, b"optimized".to_vec()); let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(test_s3_secrets()), + HashMapSecretStore::new(test_signed_route_secrets()), Arc::clone(&stub) as Arc, ); let mut settings = create_test_settings(); @@ -6380,7 +6312,7 @@ mod tests { Method::GET, "https://www.example.com/.images/foo.jpg?profile=medium", ); - let route = test_s3_image_optimizer_route(); + let route = test_signed_image_optimizer_route(); let (response, stream_body) = handle_asset_proxy_request( &settings, @@ -6390,7 +6322,7 @@ mod tests { &origin_source_of(&route), ) .await - .expect("should proxy optimized S3 asset request") + .expect("should proxy the optimized signed asset request") .into_response_and_body(); assert_eq!(response.status(), StatusCode::OK, "should proxy the asset"); @@ -6408,20 +6340,35 @@ mod tests { assert_eq!( stub.recorded_request_methods(), vec!["HEAD", "GET"], - "should preflight S3 with HEAD before the optimized GET" + "should preflight the origin with HEAD before the optimized GET" ); + let sent_uri = "https://bucket.storage.example.com/.images/foo.jpg"; assert_eq!( stub.recorded_request_uris(), + vec![sent_uri, sent_uri], + "should strip the transform query from both requests" + ); + let signatures: Vec> = stub + .recorded_request_headers() + .iter() + .map(|sent| { + sent.iter() + .find(|(name, _)| name == "authorization") + .map(|(_, value)| value.clone()) + }) + .collect(); + assert_eq!( + signatures, vec![ - "https://examplebucket.s3.us-east-1.amazonaws.com/.images/foo.jpg", - "https://examplebucket.s3.us-east-1.amazonaws.com/.images/foo.jpg", + Some(signed_route_authorization(&Method::HEAD, sent_uri)), + Some(signed_route_authorization(&Method::GET, sent_uri)), ], - "should strip transform query from both S3 requests" + "each request should be signed as its own method" ); assert_eq!( stub.recorded_stream_response_flags(), vec![false, true], - "S3 HEAD preflight stays non-streaming, final asset GET stays streaming" + "the HEAD preflight stays non-streaming, the final asset GET stays streaming" ); let options = stub.recorded_image_optimizer_options(); assert_eq!(options.len(), 2, "should send two origin requests"); @@ -6437,7 +6384,7 @@ mod tests { } #[test] - fn handle_asset_proxy_request_returns_raw_s3_error_before_image_optimizer() { + fn handle_asset_proxy_request_returns_the_origin_s_own_error_before_image_optimizer() { futures::executor::block_on(async { let stub = Arc::new(StubHttpClient::new()); stub.set_streaming_responses_supported(true); @@ -6453,7 +6400,7 @@ mod tests { ], ); let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(test_s3_secrets()), + HashMapSecretStore::new(test_signed_route_secrets()), Arc::clone(&stub) as Arc, ); let mut settings = create_test_settings(); @@ -6464,7 +6411,7 @@ mod tests { Method::GET, "https://www.example.com/.images/missing.jpg?profile=medium", ); - let route = test_s3_image_optimizer_route(); + let route = test_signed_image_optimizer_route(); let asset_response = handle_asset_proxy_request( &settings, @@ -6474,7 +6421,7 @@ mod tests { &origin_source_of(&route), ) .await - .expect("should return raw S3 error response"); + .expect("should return the origin's own error response"); assert_eq!( asset_response.cache_policy(), AssetProxyCachePolicy::NoStorePrivate, @@ -6486,40 +6433,43 @@ mod tests { assert_eq!( response_header(&response, header::CACHE_CONTROL), Some("no-store, private"), - "S3 origin errors should not be cached" + "origin errors should not be cached" ); assert!( response.headers().get(header::SET_COOKIE).is_none(), - "raw S3 error should still strip unsafe response headers" + "the origin's error should still strip unsafe response headers" ); let mut output = Vec::new(); stream_asset_body( - stream_body.expect("should preserve the raw S3 error body as a stream"), + stream_body.expect("should preserve the origin's error body as a stream"), &mut output, ) .await - .expect("should stream the raw S3 error body"); - let body = String::from_utf8(output).expect("should decode the S3 error body"); - assert!(body.contains("NoSuchKey"), "should return S3 error body"); + .expect("should stream the origin's error body"); + let body = String::from_utf8(output).expect("should decode the error body"); + assert!( + body.contains("NoSuchKey"), + "should return the origin's error body" + ); assert!( body.contains("image/upload/missing.jpg"), - "should expose the missing S3 key" + "should expose the missing key" ); assert_eq!( stub.recorded_request_methods(), vec!["HEAD", "GET"], - "should fetch S3 error body after failed HEAD preflight" + "should fetch the error body after the failed HEAD preflight" ); assert_eq!( stub.recorded_stream_response_flags(), vec![false, true], - "S3 HEAD preflight stays non-streaming, error-body GET stays streaming" + "the HEAD preflight stays non-streaming, the error-body GET stays streaming" ); let options = stub.recorded_image_optimizer_options(); assert_eq!( options, vec![None, None], - "should not invoke IO for S3 errors" + "should not invoke IO for origin errors" ); }); } @@ -6531,7 +6481,7 @@ mod tests { stub.set_streaming_responses_supported(true); stub.push_response(200, b"raw".to_vec()); let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(test_s3_secrets()), + HashMapSecretStore::new(test_signed_route_secrets()), Arc::clone(&stub) as Arc, ); let mut settings = create_test_settings(); @@ -6542,7 +6492,7 @@ mod tests { Method::GET, "https://www.example.com/.images/foo.jpg?profile=medium&_io_debug=1", ); - let route = test_s3_image_optimizer_route(); + let route = test_signed_image_optimizer_route(); let (response, stream_body) = handle_asset_proxy_request( &settings, @@ -6552,7 +6502,7 @@ mod tests { &origin_source_of(&route), ) .await - .expect("should proxy debug S3 asset request") + .expect("should proxy the debug signed asset request") .into_response_and_body(); assert_eq!(response.status(), StatusCode::OK, "should proxy the asset"); @@ -6567,7 +6517,7 @@ mod tests { assert_eq!( stub.recorded_request_methods(), vec!["GET"], - "disabled IO should not add S3 preflight" + "disabled IO should not add a signed preflight" ); assert_eq!( stub.recorded_image_optimizer_options(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 890351127a..9e5f5e094d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -91,7 +91,7 @@ use crate::response_privacy::{ }; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::{ - AUCTION_DEBUG_METADATA_ALLOWLIST, AUCTION_DEBUG_UPSTREAM_METADATA_KEYS, AssetOriginAuth, + AUCTION_DEBUG_METADATA_ALLOWLIST, AUCTION_DEBUG_UPSTREAM_METADATA_KEYS, AuctionDebugCommentFormat, AuctionDebugCommentOptions, AuctionDebugCommentVerbosity, Settings, }; use crate::source::{SourceLocation, SourceRequest}; @@ -3911,20 +3911,19 @@ const READER_HEADERS_NOT_SIGNED: [header::HeaderName; 10] = [ header::RANGE, ]; -/// Signs a publisher origin request in place with `[publisher]`'s -/// `origin_auth`, once the reader's headers a signed read must not carry are -/// removed. +/// Signs a publisher origin request in place with the source's signing, once +/// the reader's headers a signed read must not carry are removed. /// /// The request is sent to the address that was signed, the target as a parsed /// URL, so the signature and the request line agree byte for byte. fn sign_publisher_origin_request( req: &mut Request, - auth: &AssetOriginAuth, + auth: &crate::origin_auth::OriginSigning, ) -> Result<(), Report> { for name in READER_HEADERS_NOT_SIGNED { req.headers_mut().remove(name); } - crate::proxy::remove_origin_auth_namespace_headers(req.headers_mut(), auth); + auth.remove_namespace_headers(req.headers_mut()); let target = url::Url::parse(&req.uri().to_string()).change_context(TrustedServerError::Proxy { message: "invalid publisher origin uri".to_string(), @@ -3936,7 +3935,7 @@ fn sign_publisher_origin_request( message: "invalid publisher origin uri".to_string(), })?; let method = req.method().clone(); - crate::proxy::apply_asset_origin_auth(&method, &target, req.headers_mut(), auth) + auth.sign(&method, &target, req.headers_mut()) } /// Build a minimal [`AuctionContext`] for the collect phase. @@ -24090,30 +24089,36 @@ mod tests { ); } - /// The publisher origin request as `origin_url`'s path, `directory_index` - /// and `origin_auth` shape it, for an origin such as object storage. + /// The publisher origin request as the source's `url` path, + /// `directory_index` and `auth` shape it, for an origin such as object + /// storage. mod object_storage_origin_tests { use super::*; + use crate::integrations::IntegrationRegistry; + use crate::origin_auth::{OriginAuth, fixture}; use crate::platform::PlatformHttpClient; - use crate::redacted::Redacted; - use crate::s3_sigv4::{S3Credentials, sign_headers}; - use crate::settings::{OriginQueryPolicy, S3SigV4AuthConfig}; + use crate::settings::OriginQueryPolicy; const STORAGE_URL: &str = "https://storage.example.com/bucket/site"; - const REGION: &str = "test-region-1"; - const ACCESS_KEY_ID: &str = "TESTACCESSKEYID"; - const SECRET_ACCESS_KEY: &str = "test-secret-access-key"; + const SIGNING_KEY: &str = "resolved-origin-key"; const READER_AUTHORIZATION: &str = "Bearer reader-token"; - fn signing_auth(origin_query: Option) -> AssetOriginAuth { - AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: REGION.to_owned(), - secret_store: None, - access_key_id: Redacted::new(ACCESS_KEY_ID.to_owned()), - secret_access_key: Redacted::new(SECRET_ACCESS_KEY.to_owned()), - session_token: None, - origin_query, - }) + fn signing_auth(origin_query: Option) -> OriginAuth { + fixture::auth("site") + .with_setting("key_name", SIGNING_KEY) + .with_origin_query(origin_query) + } + + /// Runs the publisher proxy with the fixture signer on offer, as a + /// deployment whose build links a signer does. + async fn run_signed_publisher_proxy( + settings: &Settings, + services: &RuntimeServices, + req: HttpRequest, + ) -> PublisherResponse { + let registry = IntegrationRegistry::with_registrations(settings, &[fixture::builder()]) + .expect("should build the registry offering the fixture signer"); + run_publisher_proxy_with(settings, ®istry, services, req).await } fn reader_request(method: Method, uri: &str) -> HttpRequest { @@ -24142,7 +24147,7 @@ mod tests { .header(header::IF_UNMODIFIED_SINCE, "Tue, 01 Sep 2026 00:00:00 GMT") .header(header::IF_RANGE, "\"reader-etag\"") .header(header::RANGE, "bytes=0-99") - .header("x-amz-meta-reader", "reader-chosen") + .header("x-fixture-reader", "reader-chosen") .header(header::USER_AGENT, "reader-agent") .body(EdgeBody::empty()) .expect("should build reader request") @@ -24188,50 +24193,13 @@ mod tests { host, "storage.example.com", "should sign for the origin host" ); - let signed_at = sent_header(sent, "x-amz-date").expect("should send the signing time"); assert_eq!( sent_header(sent, "authorization"), - Some(expected_authorization(method, &uris[0], host, signed_at).as_str()), + Some(fixture::expected_authorization(SIGNING_KEY, method, &uris[0], host).as_str()), "should sign exactly the address and host that were sent" ); } - /// The `Authorization` the signer gives `method` on `uri` with `host` at - /// the moment the signed request records. - fn expected_authorization( - method: &Method, - uri: &str, - host: &str, - signed_at: &str, - ) -> String { - let signed_at = chrono::NaiveDateTime::parse_from_str(signed_at, "%Y%m%dT%H%M%SZ") - .expect("should read the signing time") - .and_utc(); - let mut headers = http::HeaderMap::new(); - headers.insert( - header::HOST, - HeaderValue::from_str(host).expect("should build host header"), - ); - sign_headers( - method, - &url::Url::parse(uri).expect("should parse sent address"), - &mut headers, - REGION, - &S3Credentials { - access_key_id: ACCESS_KEY_ID.to_owned(), - secret_access_key: Redacted::new(SECRET_ACCESS_KEY.to_owned()), - session_token: None, - }, - SystemTime::from(signed_at), - ) - .expect("should sign the expected request"); - headers - .get(header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .expect("should produce an Authorization header") - .to_owned() - } - pub(super) fn response_status(response: &PublisherResponse) -> StatusCode { match response { PublisherResponse::Buffered(response) @@ -24443,7 +24411,7 @@ mod tests { let stub = stub_with_responses(1); let services = services_for(&stub); - let _ = run_publisher_proxy( + let _ = run_signed_publisher_proxy( &settings, &services, reader_request_with_private_headers( @@ -24512,7 +24480,7 @@ mod tests { "should not send the reader's Range" ); assert_eq!( - sent_header(sent, "x-amz-meta-reader"), + sent_header(sent, "x-fixture-reader"), None, "should not send a header the reader set in the signing namespace" ); @@ -24534,7 +24502,7 @@ mod tests { let stub = stub_with_responses(1); let services = services_for(&stub); - let _ = run_publisher_proxy( + let _ = run_signed_publisher_proxy( &settings, &services, reader_request(Method::HEAD, "https://publisher.example/page"), @@ -24555,7 +24523,7 @@ mod tests { let stub = stub_with_responses(1); let services = services_for(&stub); - let _ = run_publisher_proxy( + let _ = run_signed_publisher_proxy( &settings, &services, reader_request(Method::GET, "https://publisher.example/page?utm=1"), @@ -24581,7 +24549,7 @@ mod tests { let stub = stub_with_responses(1); let services = services_for(&stub); - let response = run_publisher_proxy( + let response = run_signed_publisher_proxy( &settings, &services, reader_request(method.clone(), "https://publisher.example/page"), diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index de58560579..d5fa36ed57 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1786,14 +1786,6 @@ fn default_request_signing_enabled() -> bool { false } -fn default_s3_access_key_id() -> Redacted { - Redacted::new("access_key_id".to_string()) -} - -fn default_s3_secret_access_key() -> Redacted { - Redacted::new("secret_access_key".to_string()) -} - fn default_asset_image_optimizer_enabled() -> bool { true } @@ -1844,115 +1836,6 @@ pub enum OriginQueryPolicy { Strip, } -/// Authentication configuration for an asset origin or the publisher origin. -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] -pub enum AssetOriginAuth { - /// Sign asset origin requests with AWS Signature Version 4 for `S3`. - #[serde(rename = "s3_sigv4", alias = "s3_sig_v4")] - S3SigV4(S3SigV4AuthConfig), -} - -impl AssetOriginAuth { - pub(crate) fn normalize(&mut self) { - match self { - Self::S3SigV4(config) => config.normalize(), - } - } - - /// Refuses a table the signer cannot use, naming `setting` in the message. - pub(crate) fn prepare_runtime(&self, setting: &str) -> Result<(), Report> { - match self { - Self::S3SigV4(config) => config.prepare_runtime(setting), - } - } - - /// Return the configured origin query policy, if any. - #[must_use] - pub fn origin_query_policy(&self) -> Option { - match self { - Self::S3SigV4(config) => config.origin_query, - } - } -} - -/// AWS Signature Version 4 configuration for `S3` asset origins. -/// -/// The route `origin_url` must use the same `S3` host that `AWS` validates in -/// the `SigV4` canonical request. Credential fields hold secret-store key names -/// in app config and resolved values at runtime. -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct S3SigV4AuthConfig { - /// `AWS` region used in the credential scope. - pub region: String, - /// Deprecated per-route store selector accepted for migration only. - #[serde(default, skip_serializing)] - pub secret_store: Option, - /// Secret reference containing the `AWS` access key ID. - #[serde(default = "default_s3_access_key_id")] - pub access_key_id: Redacted, - /// Secret reference containing the `AWS` secret access key. - #[serde(default = "default_s3_secret_access_key")] - pub secret_access_key: Redacted, - /// Optional secret reference containing an `AWS` session token. - #[serde(default)] - pub session_token: Option>, - /// Query-string handling policy for the signed `S3` origin request. - /// - /// Set this to `strip` when request query parameters are transformation - /// inputs rather than `S3` object identity. If omitted, image-optimized routes - /// strip queries and plain routes preserve them. - #[serde(default)] - pub origin_query: Option, -} - -fn s3_region_is_valid(region: &str) -> bool { - region - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') -} - -impl S3SigV4AuthConfig { - fn normalize(&mut self) { - self.region = self.region.trim().to_string(); - if self.secret_store.take().is_some() { - log::warn!( - "S3 secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" - ); - } - self.access_key_id = Redacted::new(self.access_key_id.expose().trim().to_string()); - self.secret_access_key = Redacted::new(self.secret_access_key.expose().trim().to_string()); - self.session_token = self.session_token.take().and_then(|value| { - let value = value.expose().trim().to_string(); - (!value.is_empty()).then(|| Redacted::new(value)) - }); - } - - fn prepare_runtime(&self, setting: &str) -> Result<(), Report> { - if self.region.is_empty() { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("{setting} s3_sigv4 region must not be empty"), - })); - } - if !s3_region_is_valid(&self.region) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "{setting} s3_sigv4 region must contain only lowercase letters, digits, and '-'" - ), - })); - } - if self.access_key_id.expose().is_empty() || self.secret_access_key.expose().is_empty() { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "{setting} s3_sigv4 credentials must not be empty after secret resolution" - ), - })); - } - Ok(()) - } -} - /// Route-level Image Optimizer configuration for asset proxying. /// /// This block only selects the processing region and profile set. The actual @@ -2750,7 +2633,11 @@ impl ProxyAssetRoute { /// enabled Image Optimizer routes and `preserve` otherwise. #[must_use] pub fn origin_query_policy(&self) -> OriginQueryPolicy { - self.query_policy_with(self.origin_source().and_then(|origin| origin.auth).as_ref()) + self.query_policy_with( + self.origin_source() + .and_then(|origin| origin.auth) + .and_then(|auth| auth.origin_query), + ) } /// The effective origin query policy for a request to `location`, as @@ -2761,11 +2648,15 @@ impl ProxyAssetRoute { &self, location: &crate::source::SourceLocation, ) -> OriginQueryPolicy { - self.query_policy_with(location.auth()) + self.query_policy_with( + location + .auth() + .and_then(crate::origin_auth::OriginSigning::origin_query_policy), + ) } - fn query_policy_with(&self, auth: Option<&AssetOriginAuth>) -> OriginQueryPolicy { - if let Some(policy) = auth.and_then(AssetOriginAuth::origin_query_policy) { + fn query_policy_with(&self, auth: Option) -> OriginQueryPolicy { + if let Some(policy) = auth { return policy; } if let Some(policy) = self @@ -5960,9 +5851,8 @@ module = \"none\"", const CANARY_EC_PARTNER_TS_PULL_TOKEN: &str = "CANARY-EC-PARTNER-TS-PULL-TOKEN-0123456789"; const CANARY_TRUSTED_CLIENT_IP_SHARED_SECRET: &str = "CANARY-TRUSTED-CLIENT-IP-SHARED-SECRET-0123456789"; - const CANARY_S3_ACCESS_KEY_ID: &str = "CANARY-S3-ACCESS-KEY-ID-0123456789"; - const CANARY_S3_SECRET_ACCESS_KEY: &str = "CANARY-S3-SECRET-ACCESS-KEY-0123456789"; - const CANARY_S3_SESSION_TOKEN: &str = "CANARY-S3-SESSION-TOKEN-0123456789"; + const CANARY_SIGNER_KEY: &str = "CANARY-SIGNER-KEY-0123456789"; + const CANARY_SIGNER_TOKEN: &str = "CANARY-SIGNER-TOKEN-0123456789"; const CANARY_TINYBIRD_AUCTION_TOKEN: &str = "CANARY-TINYBIRD-AUCTION-TOKEN-0123456789"; const CANARY_TINYBIRD_ACCESS_TOKEN: &str = "CANARY-TINYBIRD-ACCESS-TOKEN-0123456789"; const CANARY_MODULE_KEY: &str = "CANARY-MODULE-TABLE-KEY-0123456789"; @@ -5993,17 +5883,15 @@ module = \"none\"", shared_secret: Redacted::new(CANARY_TRUSTED_CLIENT_IP_SHARED_SECRET.to_string()), }); - let mut asset_route = ProxyAssetRoute::new("/s3-assets/", "https://s3.canary.example"); + let mut asset_route = + ProxyAssetRoute::new("/signed-assets/", "https://signed.canary.example"); asset_route.set_origin_source(&crate::source::OriginSettings { - auth: Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "us-east-1".to_string(), - secret_store: None, - access_key_id: Redacted::new(CANARY_S3_ACCESS_KEY_ID.to_string()), - secret_access_key: Redacted::new(CANARY_S3_SECRET_ACCESS_KEY.to_string()), - session_token: Some(Redacted::new(CANARY_S3_SESSION_TOKEN.to_string())), - origin_query: None, - })), - ..crate::source::OriginSettings::new("https://s3.canary.example") + auth: Some( + crate::origin_auth::fixture::auth("canary") + .with_setting("key_name", CANARY_SIGNER_KEY) + .with_setting("token_name", CANARY_SIGNER_TOKEN), + ), + ..crate::source::OriginSettings::new("https://signed.canary.example") }); settings.proxy.asset_routes = vec![asset_route]; @@ -6061,16 +5949,12 @@ module = \"none\"", CANARY_TRUSTED_CLIENT_IP_SHARED_SECRET, ), ( - "proxy.asset_routes[].auth.access_key_id", - CANARY_S3_ACCESS_KEY_ID, + "proxy.asset_routes[].source.origin.auth.key_name", + CANARY_SIGNER_KEY, ), ( - "proxy.asset_routes[].auth.secret_access_key", - CANARY_S3_SECRET_ACCESS_KEY, - ), - ( - "proxy.asset_routes[].auth.session_token", - CANARY_S3_SESSION_TOKEN, + "proxy.asset_routes[].source.origin.auth.token_name", + CANARY_SIGNER_TOKEN, ), ( "analytics.tinybird.auction_token_secret", @@ -8608,7 +8492,10 @@ host_header = "www.example.com:8443""#, settings .origin_source() .expect("should select the origin source") - .location() + .location( + &crate::origin_auth::OriginSigners::default(), + "source.origin" + ) .expect("should give a location") .host_header(), "www.example.com:8443", @@ -9313,7 +9200,10 @@ source_domain = "partner.example.com" assert_eq!(origin.host_header.as_deref(), Some("www.example.com")); assert_eq!( origin - .location() + .location( + &crate::origin_auth::OriginSigners::default(), + "source.origin" + ) .expect("should give a location") .host_header(), "www.example.com" @@ -9417,10 +9307,10 @@ source_domain = "partner.example.com" } #[test] - fn source_origin_auth_reads_the_asset_route_auth_table() { + fn source_origin_auth_reads_its_type_and_query_policy_and_keeps_the_rest() { let settings = Settings::from_toml(&origin_settings_with( STORAGE_ORIGIN_URL, - r#"auth = { type = "s3_sigv4", region = "test-region-1", access_key_id = "origin_access_key_id", secret_access_key = "origin_secret_access_key", session_token = "origin_session_token", origin_query = "strip" } + r#"auth = { type = " fixture_scheme ", realm = "site", key_name = "origin_key", origin_query = "strip" } directory_index = "index.html""#, )) .expect("should parse an origin with auth and directory_index"); @@ -9428,52 +9318,43 @@ source_domain = "partner.example.com" .origin_source() .expect("should select the origin source"); - let Some(AssetOriginAuth::S3SigV4(auth)) = origin.auth.as_ref() else { - panic!("should read auth as the asset route auth table"); - }; - assert_eq!(auth.region, "test-region-1"); - assert_eq!(auth.access_key_id.expose(), "origin_access_key_id"); - assert_eq!(auth.secret_access_key.expose(), "origin_secret_access_key"); + let auth = origin.auth.as_ref().expect("should read auth"); + assert_eq!(auth.signer, "fixture_scheme", "the type should be trimmed"); + assert_eq!(auth.origin_query, Some(OriginQueryPolicy::Strip)); assert_eq!( - auth.session_token - .as_ref() - .map(Redacted::expose) - .map(String::as_str), - Some("origin_session_token") + auth.settings.get("realm").and_then(JsonValue::as_str), + Some("site") + ); + assert_eq!( + auth.settings.get("key_name").and_then(JsonValue::as_str), + Some("origin_key") ); assert!( - origin - .location() - .expect("should give a location") - .query_is_stripped(), - "origin_query = \"strip\" should leave the query off origin requests" + !auth.settings.contains_key("type") && !auth.settings.contains_key("origin_query"), + "the keys core reads should stay out of the signer's settings" ); assert_eq!(origin.directory_index.as_deref(), Some("index.html")); - let defaulted = Settings::from_toml(&origin_settings_with( + let location = origin + .location(&crate::origin_auth::fixture::signers(), "source.origin") + .expect("should give a location signing through the fixture"); + assert!( + location.query_is_stripped(), + "origin_query = \"strip\" should leave the query off origin requests" + ); + + let kept = Settings::from_toml(&origin_settings_with( STORAGE_ORIGIN_URL, - r#"auth = { type = "s3_sigv4", region = "test-region-1" }"#, + r#"auth = { type = "fixture_scheme", realm = "site" }"#, )) - .expect("should parse auth naming only its region"); - let defaulted = defaulted + .expect("should parse auth naming only its type and realm"); + let kept = kept .origin_source() - .expect("should select the origin source"); - let Some(AssetOriginAuth::S3SigV4(auth)) = defaulted.auth.as_ref() else { - panic!("should read auth with defaulted key references"); - }; - assert_eq!( - ( - auth.access_key_id.expose().as_str(), - auth.secret_access_key.expose().as_str() - ), - ("access_key_id", "secret_access_key"), - "omitted key references should default as an asset route's do" - ); + .expect("should select the origin source") + .location(&crate::origin_auth::fixture::signers(), "source.origin") + .expect("should give a location"); assert!( - !defaulted - .location() - .expect("should give a location") - .query_is_stripped(), + !kept.query_is_stripped(), "the query should be kept unless origin_query strips it" ); } @@ -9538,44 +9419,29 @@ source_domain = "partner.example.com" } #[test] - fn source_origin_auth_refuses_a_table_it_cannot_sign_with() { + fn source_origin_auth_refuses_a_table_core_cannot_use() { for (origin_url, table, expected) in [ ( - STORAGE_ORIGIN_URL, - r#"{ type = "s3_sigv4", region = "" }"#, - "source.origin.auth s3_sigv4 region must not be empty", - ), - ( - STORAGE_ORIGIN_URL, - r#"{ type = "s3_sigv4", region = "Test_Region" }"#, - "source.origin.auth s3_sigv4 region must contain only lowercase letters", - ), - ( - STORAGE_ORIGIN_URL, - r#"{ type = "s3_sigv4", region = "test-region-1", access_key_id = " " }"#, - "source.origin.auth s3_sigv4 credentials must not be empty", + "http://storage.example.com/bucket/site", + r#"{ type = "fixture_scheme", realm = "site" }"#, + "[source.origin] signs with auth, so url must use https", ), ( STORAGE_ORIGIN_URL, - r#"{ type = "s3_sigv4", region = "test-region-1", secret_access_key = "" }"#, - "source.origin.auth s3_sigv4 credentials must not be empty", - ), - ( - "http://storage.example.com/bucket/site", - r#"{ type = "s3_sigv4", region = "test-region-1" }"#, - "[source.origin] signs with auth, so url must use https", + r#"{ realm = "site" }"#, + "[source.origin] could not be read", ), ( STORAGE_ORIGIN_URL, - r#"{ type = "unknown_scheme", region = "test-region-1" }"#, - "source.origin", + r#"{ type = "" }"#, + "[source.origin.auth] names no signer", ), ( STORAGE_ORIGIN_URL, - r#"{ type = "s3_sigv4", region = "test-region-1", bucket = "site" }"#, - "bucket", + r#"{ type = "Not A Name" }"#, + "[source.origin.auth] type `Not A Name` is not a signer's name", ), - (STORAGE_ORIGIN_URL, r#""s3_sigv4""#, "source.origin"), + (STORAGE_ORIGIN_URL, r#""fixture_scheme""#, "source.origin"), ] { let err = Settings::from_toml(&origin_settings_with( origin_url, @@ -9593,44 +9459,33 @@ source_domain = "partner.example.com" fn asset_route_auth_refusals_name_the_asset_route() { let mut route = ProxyAssetRoute::new("/assets/", "https://storage.example.com"); route.set_origin_source(&crate::source::OriginSettings { - auth: Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: String::new(), - secret_store: None, - access_key_id: Redacted::new("access_key_id".to_owned()), - secret_access_key: Redacted::new("secret_access_key".to_owned()), - session_token: None, - origin_query: None, - })), + auth: Some(crate::origin_auth::OriginAuth::new("")), ..crate::source::OriginSettings::new("https://storage.example.com") }); let err = route .prepare_runtime() - .expect_err("should refuse an asset route auth with no region"); + .expect_err("should refuse an asset route auth naming no signer"); assert!( - format!("{err:?}").contains( - "proxy.asset_routes /assets/ source.origin.auth s3_sigv4 region must not be empty" - ), + format!("{err:?}") + .contains("[proxy.asset_routes /assets/ source.origin.auth] names no signer"), "the asset route refusal should name the route: {err:?}" ); } #[test] - fn source_origin_auth_credentials_stay_out_of_debug_output() { + fn source_origin_auth_settings_stay_out_of_debug_output() { let canaries = [ - "canary-origin-access", - "canary-origin-secret", - "canary-origin-session", + "canary-origin-realm", + "canary-origin-key", + "canary-origin-token", ]; let origin = crate::source::OriginSettings { - auth: Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "test-region-1".to_owned(), - secret_store: None, - access_key_id: Redacted::new(canaries[0].to_owned()), - secret_access_key: Redacted::new(canaries[1].to_owned()), - session_token: Some(Redacted::new(canaries[2].to_owned())), - origin_query: None, - })), + auth: Some( + crate::origin_auth::fixture::auth(canaries[0]) + .with_setting("key_name", canaries[1]) + .with_setting("token_name", canaries[2]), + ), ..crate::source::OriginSettings::new("https://storage.example.com/bucket/site") }; let mut settings = Settings::default(); @@ -9639,8 +9494,8 @@ source_domain = "partner.example.com" let debug = format!("{origin:?} {settings:?}"); assert!( - debug.contains("test-region-1"), - "should leave the region visible in debug output: {debug}" + debug.contains("fixture_scheme"), + "should leave the signer's name visible in debug output: {debug}" ); for canary in canaries { assert!( @@ -9900,7 +9755,7 @@ source_domain = "partner.example.com" "[source.origin] directory_index", ), ( - r#"origin_auth = { type = "s3_sigv4", region = "test-region-1" }"#, + r#"origin_auth = { type = "fixture_scheme", realm = "site" }"#, "[source.origin] auth", ), ] { @@ -9929,7 +9784,7 @@ source_domain = "partner.example.com" ( r#"source.module = "origin" source.origin.url = "https://assets.example.com" - auth = { type = "s3_sigv4", region = "us-east-1" }"#, + auth = { type = "fixture_scheme", realm = "assets" }"#, "[proxy.asset_routes.source.origin] auth", ), ] { @@ -10334,11 +10189,11 @@ source_domain = "partner.example.com" [[proxy.asset_routes]] prefix = "/.image/" source.module = "origin" - source.origin.url = "https://bucket.s3.us-east-1.amazonaws.com" + source.origin.url = "https://bucket.storage.example.com" [proxy.asset_routes.source.origin.auth] - type = "s3_sigv4" - region = "us-east-1" + type = "fixture_scheme" + realm = "images" origin_query = "strip" [proxy.asset_routes.image_optimizer] @@ -10348,82 +10203,25 @@ source_domain = "partner.example.com" "#; let settings = Settings::from_toml(&toml_str) - .expect("should parse S3 auth and image optimizer asset route"); + .expect("should parse a signed image optimizer asset route"); let route = settings .asset_route_for_path("/.image/id/example.jpg") .expect("should match configured route"); assert!(route.image_optimizer_enabled()); assert_eq!(route.origin_query_policy(), OriginQueryPolicy::Strip); - match route + let auth = route .origin_source() .and_then(|origin| origin.auth) - .expect("should configure route auth") - { - AssetOriginAuth::S3SigV4(config) => { - assert_eq!(config.region, "us-east-1"); - assert_eq!(config.secret_store, None); - assert_eq!(config.access_key_id.expose(), "access_key_id"); - assert_eq!(config.secret_access_key.expose(), "secret_access_key"); - } - } - } - - #[test] - fn proxy_asset_route_validation_rejects_s3_sigv4_http_origin_url() { - let toml_str = crate_test_settings_str() - + r#" - [proxy] - - [[proxy.asset_routes]] - prefix = "/.images/" - source.module = "origin" - source.origin.url = "http://bucket.s3.us-east-1.amazonaws.com" - - [proxy.asset_routes.source.origin.auth] - type = "s3_sigv4" - region = "us-east-1" - "#; - - let err = Settings::from_toml(&toml_str) - .expect_err("should reject cleartext S3 SigV4 origin URLs"); - - assert!( - format!("{err:?}").contains("signs with auth, so url must use https"), - "should mention the S3 SigV4 HTTPS requirement: {err:?}" + .expect("should configure route auth"); + assert_eq!(auth.signer, "fixture_scheme"); + assert_eq!( + auth.settings.get("realm").and_then(JsonValue::as_str), + Some("images") ); } #[test] - fn proxy_asset_route_validation_rejects_invalid_s3_regions() { - for region in ["us east 1", "us/east/1", "US-EAST-1", "us-east-\\n1"] { - let toml_str = crate_test_settings_str() - + &format!( - r#" - [proxy] - - [[proxy.asset_routes]] - prefix = "/.images/" - source.module = "origin" - source.origin.url = "https://bucket.s3.us-east-1.amazonaws.com" - - [proxy.asset_routes.source.origin.auth] - type = "s3_sigv4" - region = "{region}" - "# - ); - - let err = Settings::from_toml(&toml_str) - .expect_err("should reject malformed S3 region values"); - - assert!( - format!("{err:?}").contains("region must contain only lowercase letters"), - "should mention the S3 region character policy for {region:?}: {err:?}" - ); - } - } - - #[test] - fn proxy_asset_route_validation_rejects_unknown_s3_auth_fields() { + fn proxy_asset_route_validation_rejects_a_signed_http_origin_url() { let toml_str = crate_test_settings_str() + r#" [proxy] @@ -10431,20 +10229,19 @@ source_domain = "partner.example.com" [[proxy.asset_routes]] prefix = "/.images/" source.module = "origin" - source.origin.url = "https://bucket.s3.us-east-1.amazonaws.com" + source.origin.url = "http://bucket.storage.example.com" [proxy.asset_routes.source.origin.auth] - type = "s3_sigv4" - region = "us-east-1" - secret_access_key_name = "secret_access_key" + type = "fixture_scheme" + realm = "images" "#; let err = Settings::from_toml(&toml_str) - .expect_err("should reject unknown S3 auth config fields"); + .expect_err("should reject a signed origin reached over cleartext"); assert!( - format!("{err:?}").contains("secret_access_key_name"), - "should mention the unknown S3 auth field: {err:?}" + format!("{err:?}").contains("signs with auth, so url must use https"), + "should mention the https requirement: {err:?}" ); } @@ -10771,19 +10568,19 @@ source_domain = "partner.example.com" format!( "{ENVIRONMENT_VARIABLE_PREFIX}{separator}PROXY{separator}ASSET_ROUTES{separator}0{separator}SOURCE{separator}ORIGIN{separator}URL" ), - Some("https://bucket.s3.us-west-2.amazonaws.com"), + Some("https://bucket.storage.example.com"), ), ( format!( "{ENVIRONMENT_VARIABLE_PREFIX}{separator}PROXY{separator}ASSET_ROUTES{separator}0{separator}SOURCE{separator}ORIGIN{separator}AUTH{separator}TYPE" ), - Some("s3_sigv4"), + Some("fixture_scheme"), ), ( format!( - "{ENVIRONMENT_VARIABLE_PREFIX}{separator}PROXY{separator}ASSET_ROUTES{separator}0{separator}SOURCE{separator}ORIGIN{separator}AUTH{separator}REGION" + "{ENVIRONMENT_VARIABLE_PREFIX}{separator}PROXY{separator}ASSET_ROUTES{separator}0{separator}SOURCE{separator}ORIGIN{separator}AUTH{separator}REALM" ), - Some("us-west-2"), + Some("images"), ), ( format!( @@ -10911,7 +10708,7 @@ source_domain = "partner.example.com" [[proxy.asset_routes]] prefix = "/.image/" source.module = "origin" - source.origin.url = "https://bucket.s3.us-east-1.amazonaws.com" + source.origin.url = "https://bucket.storage.example.com" [proxy.asset_routes.image_optimizer] enabled = true diff --git a/crates/trusted-server-core/src/source.rs b/crates/trusted-server-core/src/source.rs index d2925f8e35..f94f0342dd 100644 --- a/crates/trusted-server-core/src/source.rs +++ b/crates/trusted-server-core/src/source.rs @@ -21,6 +21,8 @@ //! Core supplies one source, `origin`, an HTTP origin named in its table. A //! module offers another through //! [`IntegrationRegistrationBuilder::with_source_module`](crate::integrations::IntegrationRegistrationBuilder::with_source_module). +//! An origin that answers only signed reads is signed through the signer its +//! `auth` table names, see [`crate::origin_auth`]. use std::fmt; use std::sync::Arc; @@ -32,9 +34,10 @@ use serde_json::{Map, Value}; use url::Url; use crate::error::TrustedServerError; +use crate::origin_auth::{OriginAuth, OriginSigners, OriginSigning}; use crate::provider_table::SectionModules; use crate::redacted::{Redacted, SENSITIVE_MARKER}; -use crate::settings::{AssetOriginAuth, OriginQueryPolicy}; +use crate::settings::OriginQueryPolicy; /// The section that selects the publisher's source. pub const SOURCE_SECTION: &str = "source"; @@ -66,7 +69,7 @@ pub struct SourceLocation { url: String, parsed: Url, host_header: Option, - auth: Option, + auth: Option, directory_index: Option, } @@ -110,7 +113,7 @@ impl SourceLocation { /// The same location signing each request with `auth`. #[must_use] - pub fn with_auth(mut self, auth: Option) -> Self { + pub fn with_auth(mut self, auth: Option) -> Self { self.auth = auth; self } @@ -159,7 +162,7 @@ impl SourceLocation { /// The signing applied to each request, if any. #[must_use] - pub fn auth(&self) -> Option<&AssetOriginAuth> { + pub fn auth(&self) -> Option<&OriginSigning> { self.auth.as_ref() } @@ -175,7 +178,7 @@ impl SourceLocation { pub fn query_is_stripped(&self) -> bool { self.auth .as_ref() - .and_then(AssetOriginAuth::origin_query_policy) + .and_then(OriginSigning::origin_query_policy) == Some(OriginQueryPolicy::Strip) } @@ -237,9 +240,10 @@ pub struct OriginSettings { #[serde(default, skip_serializing_if = "Option::is_none")] pub directory_index: Option, /// Signing of every request, for an origin that answers only signed - /// reads. + /// reads: the signer's name as `type` and its settings beside it, see + /// [`crate::origin_auth`]. #[serde(default, skip_serializing_if = "Option::is_none")] - pub auth: Option, + pub auth: Option, } impl fmt::Debug for OriginSettings { @@ -300,12 +304,14 @@ impl OriginSettings { Ok(origin) } - /// Reads and checks the table at `setting`. + /// Reads and checks the table at `setting`. The signer `auth` names is + /// checked where the source is built, because only the deployment's + /// builders know the signers on offer. /// /// # Errors /// /// When the table cannot be read, or names an address, a `Host`, an index - /// file or signing core cannot use, naming `setting`. + /// file or an `auth` core cannot use, naming `setting`. pub fn read( settings: &Map, setting: &str, @@ -334,7 +340,7 @@ impl OriginSettings { if parsed.scheme() != "https" { return Err(refuse("signs with auth, so url must use https".to_owned())); } - auth.prepare_runtime(&format!("{setting}.auth"))?; + auth.check(&format!("{setting}.auth"))?; } Ok(()) } @@ -345,16 +351,28 @@ impl OriginSettings { crate::settings::Publisher::is_placeholder_origin_url(&self.url) } - /// The location these settings name. + /// The location these settings name, signing through the signer + /// `signers` offers under `auth`'s type, where `setting` names the table + /// in a refusal. /// /// # Errors /// /// When the address cannot be read, which [`Self::read`] has already - /// refused for settings it returned. - pub fn location(&self) -> Result> { + /// refused for settings it returned, when `auth` names a signer nothing + /// offers, or when the signer cannot use its table. + pub fn location( + &self, + signers: &OriginSigners, + setting: &str, + ) -> Result> { + let auth = self + .auth + .as_ref() + .map(|auth| signers.build(auth, &format!("{setting}.auth"))) + .transpose()?; Ok(SourceLocation::new(self.url.clone())? .with_host_header(self.host_header.clone()) - .with_auth(self.auth.clone()) + .with_auth(auth) .with_directory_index(self.directory_index.clone())) } } @@ -367,14 +385,19 @@ pub struct OriginSource { } impl OriginSource { - /// The source `settings` name. + /// The source `settings` name, signing through `signers`, where `setting` + /// names the table in a refusal. /// /// # Errors /// - /// When the settings' address cannot be read. - pub fn new(settings: &OriginSettings) -> Result> { + /// As [`OriginSettings::location`]. + pub fn new( + settings: &OriginSettings, + signers: &OriginSigners, + setting: &str, + ) -> Result> { Ok(Self { - location: settings.location()?, + location: settings.location(signers, setting)?, }) } } @@ -388,9 +411,20 @@ impl Source for OriginSource { } } -/// Builds [`OriginSource`] from an `origin` table. -#[derive(Debug, Clone, Copy, Default)] -pub struct OriginFactory; +/// Builds [`OriginSource`] from an `origin` table, signing through the +/// signers it was given. +#[derive(Debug, Clone, Default)] +pub struct OriginFactory { + signers: OriginSigners, +} + +impl OriginFactory { + /// A factory whose sources sign through `signers`. + #[must_use] + pub fn new(signers: OriginSigners) -> Self { + Self { signers } + } +} impl SourceFactory for OriginFactory { fn build( @@ -399,7 +433,11 @@ impl SourceFactory for OriginFactory { setting: &str, ) -> Result, Report> { let origin = OriginSettings::read(settings, setting)?; - Ok(Arc::new(OriginSource::new(&origin)?)) + Ok(Arc::new(OriginSource::new( + &origin, + &self.signers, + setting, + )?)) } } @@ -610,9 +648,17 @@ mod tests { "[source.origin] directory_index", ), ( - json!({ "url": "http://origin.example.com", "auth": { "type": "s3_sigv4", "region": "test-region-1" } }), + json!({ "url": "http://origin.example.com", "auth": { "type": "fixture_scheme", "realm": "site" } }), "[source.origin] signs with auth, so url must use https", ), + ( + json!({ "url": "https://origin.example.com", "auth": { "realm": "site" } }), + "[source.origin] could not be read", + ), + ( + json!({ "url": "https://origin.example.com", "auth": { "type": " " } }), + "[source.origin.auth] names no signer", + ), ] { let err = OriginSettings::read(&table(value.clone()), "source.origin") .expect_err("should refuse the table"); @@ -650,7 +696,7 @@ mod tests { #[test] fn the_origin_source_answers_the_same_location_for_every_request() { - let source = OriginFactory + let source = OriginFactory::default() .build( &table(json!({ "url": "https://origin.example.com" })), "source.origin", @@ -668,4 +714,69 @@ mod tests { .expect("should locate"); assert_eq!(location.url(), "https://origin.example.com"); } + + #[test] + fn the_origin_source_signs_through_the_signer_its_auth_names() { + use crate::origin_auth::fixture; + + let settings = table(json!({ + "url": "https://storage.example.com/site", + "auth": { "type": fixture::SIGNER, "realm": "site", "origin_query": "strip" }, + })); + + let err = OriginFactory::default() + .build(&settings, "source.origin") + .expect_err("should refuse the signer with none on offer"); + assert!( + format!("{err:?}").contains( + "[source.origin.auth] type `fixture_scheme` names an origin signer this \ + deployment does not offer" + ), + "{err:?}" + ); + + let source = OriginFactory::new(fixture::signers()) + .build(&settings, "source.origin") + .expect("should build with the fixture on offer"); + let headers = HeaderMap::new(); + let location = source + .locate(&SourceRequest { + method: &Method::GET, + path: "/page", + query: "", + headers: &headers, + removed_prefix: None, + }) + .expect("should locate"); + assert!( + location.query_is_stripped(), + "the auth table strips the query" + ); + let mut sent = HeaderMap::new(); + sent.insert( + http::header::HOST, + http::HeaderValue::from_static("storage.example.com"), + ); + location + .auth() + .expect("should sign") + .sign(&Method::GET, location.parsed(), &mut sent) + .expect("should sign the request"); + assert!(sent.contains_key(http::header::AUTHORIZATION)); + + let err = OriginFactory::new(fixture::signers()) + .build( + &table(json!({ + "url": "https://storage.example.com/site", + "auth": { "type": fixture::SIGNER, "realm": "" }, + })), + "source.origin", + ) + .expect_err("should refuse a table the signer cannot use"); + assert!( + format!("{err:?}") + .contains("[source.origin.auth] fixture_scheme realm must not be empty"), + "{err:?}" + ); + } } diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 12d1e27164..83e60a5203 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -29,7 +29,7 @@ pub mod tests { /// When `route` does not select core's `origin` source. pub fn set_route_auth( route: &mut crate::settings::ProxyAssetRoute, - auth: Option, + auth: Option, ) { let mut origin = route .origin_source() @@ -46,10 +46,10 @@ pub mod tests { /// When `settings` do not select core's `origin` source. #[must_use] pub fn origin_host(settings: &Settings) -> String { - settings + let origin = settings .origin_source() - .expect("settings should select the origin source") - .location() + .expect("settings should select the origin source"); + crate::source::SourceLocation::new(origin.url) .expect("the origin source should give a location") .host() } diff --git a/crates/trusted-server-modules/Cargo.toml b/crates/trusted-server-modules/Cargo.toml index 2e37fab8d3..4588d84d4c 100644 --- a/crates/trusted-server-modules/Cargo.toml +++ b/crates/trusted-server-modules/Cargo.toml @@ -31,6 +31,7 @@ trusted-server-core = { workspace = true } trusted-server-framework-nextjs = { workspace = true } trusted-server-identity-lockr = { workspace = true } trusted-server-loading-cache-51degrees = { workspace = true } +trusted-server-shared-aws = { workspace = true } trusted-server-tag-google-tag = { workspace = true } trusted-server-tag-google-tag-manager = { workspace = true } trusted-server-testing-testlight = { workspace = true } diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index 0b6da8f243..bd2089dc91 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -51,6 +51,9 @@ pub fn builders() -> Vec { // The loading cache a stock build caches with, offered whatever the // settings say, which no section selects either. trusted_server_loading_cache_51degrees::builder(), + // The origin signer a source's auth table can name, offered the same + // way. + trusted_server_shared_aws::builder(), ] } @@ -504,6 +507,7 @@ middleware = [\"robots_txt\"] "auction.aps", "ad-server.mock", "loading-cache.51degrees", + "shared.aws", ], "should offer the stock modules in the order their hooks run" ); diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 181fa9833a..64812e563c 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -1022,8 +1022,9 @@ https://edge.example.com/_ts/debug/ja4`. ### GET or HEAD `{path}` **Contract:** Fastly-only and registered once for each -`[[proxy.asset_routes]]` entry. Auth: optional S3 SigV4 origin credentials, -which authenticate Trusted Server to the origin rather than the caller. +`[[proxy.asset_routes]]` entry. Auth: optional origin signing by the signer +the route's `auth` table names, which authenticates Trusted Server to the +origin rather than the caller. Request body: not applicable. The route maps the remainder of the path and the configured query policy to the selected origin, preserves range and conditional request semantics, and returns the origin status/body after stripping unsafe diff --git a/docs/guide/asset-routes.md b/docs/guide/asset-routes.md index 8499812d28..9d013969de 100644 --- a/docs/guide/asset-routes.md +++ b/docs/guide/asset-routes.md @@ -65,9 +65,9 @@ url = "https://assets-cdn.example.com" Both fields must be configured together. The rewritten path must start with `/`. When `url` has a path, the rewritten path is added under that folder. -## Private S3 origins +## Signed origins -Add an auth block to the route's source when the asset origin is a private S3 bucket. +Add an auth block to the route's source when the asset origin answers only signed reads. `type` names an origin signer a crate in the build offers, and the rest of the table is that signer's. A stock build offers `s3_sigv4`, from `crates/shared/aws`, for a private S3 bucket. ```toml [[proxy.asset_routes]] @@ -91,7 +91,7 @@ secret_access_key = "s3_secret_access_key" ### S3 requirements - `url` must be the real S3 host that AWS validates in the SigV4 canonical request. -- `url` must use `https` when the route signs. A signing table the signer cannot use, such as an empty or malformed `region`, is refused at startup naming the route, as in `proxy.asset_routes /.image/ source.origin.auth`. +- `url` must use `https` when the route signs. A `type` no crate in the build offers is refused at startup naming the signers on offer, and a signing table the signer cannot use, such as an empty or malformed `region`, is refused at startup and by `ts config validate` naming the route, as in `proxy.asset_routes /.image/ source.origin.auth`. - S3 support is for `GET` and `HEAD` asset reads. - Signing uses header-based AWS SigV4, not presigned URLs. - The signer uses `x-amz-content-sha256: UNSIGNED-PAYLOAD`. @@ -109,7 +109,7 @@ Credential fields contain secret key references: | `secret_access_key` | `secret_access_key` | AWS secret access key | | `session_token` | unset | Optional AWS temporary session token | -Place those values in the logical `trusted_server_secrets` store. Adapter configuration maps that logical ID to an environment-specific physical store such as Fastly `ts_secrets`. The legacy `secret_store` field is accepted for one migration release but ignored and omitted from newly pushed config. +Place those values in the logical `trusted_server_secrets` store. Adapter configuration maps that logical ID to an environment-specific physical store such as Fastly `ts_secrets`. The legacy `secret_store` field is accepted by the signer and ignored. ## Origin query policy diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 0ff62313a1..eefbec7996 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -103,7 +103,7 @@ Two accepted secret-shaped fields are deliberately different: `analytics.tinybird.auction_token_secret` as the store key name instead. The four deprecated `secret_store` selectors under Tinybird, DataDome, its -protection-test bypass, and S3 route authentication are also accepted and +protection-test bypass, and the `s3_sigv4` origin signer are also accepted and discarded. Store selection comes from the adapter's EdgeZero mapping. ::: warning CLI output and inline secrets @@ -116,25 +116,24 @@ diff; it does not move inline values to a secret store. The following table records the independent lifecycle, key-identity, serialization, runtime, and secret axes for every exceptional field: -| Path | Lifecycle | Key identity | Serialization | Runtime | Secret handling | -| ------------------------------------------------------------ | ---------- | ----------------------------------- | ------------- | ----------------------- | ------------------------ | -| `AssetOriginAuth.s3_sig_v4` | deprecated | alias of `AssetOriginAuth.s3_sigv4` | skipped | deserialization only | none | -| `DataDomeConfig.server_side_key_secret_name` | canonical | canonical | serialized | active | store resolved | -| `DataDomeConfig.server_side_key_secret_store` | deprecated | canonical | serialized | discarded by the module | none | -| `DataDomeProtectionTestBypassConfig.credential_secret_name` | canonical | canonical | serialized | active | store resolved | -| `DataDomeProtectionTestBypassConfig.credential_secret_store` | deprecated | canonical | serialized | discarded by the module | none | -| `Ec.passphrase` | canonical | canonical | serialized | active | store resolved | -| `EcPartner.api_token` | canonical | canonical | serialized | active | store resolved | -| `EcPartner.ts_pull_token` | canonical | canonical | serialized | active | store resolved | -| `Publisher.proxy_secret` | canonical | canonical | serialized | active | store resolved | -| `S3SigV4AuthConfig.access_key_id` | canonical | canonical | serialized | active | store resolved | -| `S3SigV4AuthConfig.secret_access_key` | canonical | canonical | serialized | active | store resolved | -| `S3SigV4AuthConfig.secret_store` | deprecated | canonical | skipped | normalized away | none | -| `S3SigV4AuthConfig.session_token` | canonical | canonical | serialized | active | store resolved | -| `TinybirdSettings.access_token_secret` | deprecated | canonical | skipped | normalized away | accepted, then discarded | -| `TinybirdSettings.auction_token_secret` | canonical | canonical | serialized | active | store resolved | -| `TinybirdSettings.secret_store` | deprecated | canonical | skipped | normalized away | none | -| `TrustedClientIpConfig.shared_secret` | canonical | canonical | serialized | active | deliberately inline | +| Path | Lifecycle | Key identity | Serialization | Runtime | Secret handling | +| ------------------------------------------------------------ | ---------- | ------------ | ------------- | ----------------------- | ------------------------ | +| `DataDomeConfig.server_side_key_secret_name` | canonical | canonical | serialized | active | store resolved | +| `DataDomeConfig.server_side_key_secret_store` | deprecated | canonical | serialized | discarded by the module | none | +| `DataDomeProtectionTestBypassConfig.credential_secret_name` | canonical | canonical | serialized | active | store resolved | +| `DataDomeProtectionTestBypassConfig.credential_secret_store` | deprecated | canonical | serialized | discarded by the module | none | +| `Ec.passphrase` | canonical | canonical | serialized | active | store resolved | +| `EcPartner.api_token` | canonical | canonical | serialized | active | store resolved | +| `EcPartner.ts_pull_token` | canonical | canonical | serialized | active | store resolved | +| `Publisher.proxy_secret` | canonical | canonical | serialized | active | store resolved | +| `s3_sigv4.access_key_id` | canonical | canonical | serialized | active | store resolved | +| `s3_sigv4.secret_access_key` | canonical | canonical | serialized | active | store resolved | +| `s3_sigv4.secret_store` | deprecated | canonical | serialized | discarded by the signer | none | +| `s3_sigv4.session_token` | canonical | canonical | serialized | active | store resolved | +| `TinybirdSettings.access_token_secret` | deprecated | canonical | skipped | normalized away | accepted, then discarded | +| `TinybirdSettings.auction_token_secret` | canonical | canonical | serialized | active | store resolved | +| `TinybirdSettings.secret_store` | deprecated | canonical | skipped | normalized away | none | +| `TrustedClientIpConfig.shared_secret` | canonical | canonical | serialized | active | deliberately inline | Prepare an initial reference-based deployment in this order: @@ -642,8 +641,10 @@ the page pipeline, which an [asset route](#proxy-asset-routes) skips. **Format**: The same table as [`[proxy.asset_routes.source.origin.auth]`](#proxy-asset-routes-source-origin-auth), with the same fields, defaults and secret-store key references, signed by the -same signer. Write `[source.origin.auth]` after the other `[source.origin]` -keys, because a table header ends the keys of the table before it. +same signer. `type` names the signer and `origin_query` is read by Trusted +Server, and every other key belongs to the signer named. Write +`[source.origin.auth]` after the other `[source.origin]` keys, because a table +header ends the keys of the table before it. ```toml [source] @@ -673,8 +674,10 @@ secret_access_key = "publisher_origin_secret_access_key" the signing scheme's own header namespace. - `origin_query = "strip"` leaves the reader's query off the origin request. By default the query is kept. -- `url` must use `https`. A table the signer cannot use, such as an empty or - malformed `region`, is refused at startup naming `source.origin.auth`. +- `url` must use `https`. A `type` no crate in the build offers is refused at + startup naming the signers on offer, and a table the signer cannot use, such + as an empty or malformed `region`, is refused at startup and by + `ts config validate` naming `source.origin.auth`. **Default**: Unset, and origin requests are sent unsigned with the reader's headers. @@ -1669,11 +1672,14 @@ beneath. Core's `origin` source takes these settings: ### `[proxy.asset_routes.source.origin.auth]` -The first supported origin auth type is `s3_sigv4`. +`type` names an origin signer a crate in the build offers, and `origin_query` +says whether the reader's query is kept on the signed request. Every other +key in the table belongs to the signer named. A stock build offers +`s3_sigv4`, from `crates/shared/aws`, whose table is: | Field | Type | Required | Default | Description | | ------------------- | ------ | -------- | ------------------- | ------------------------------------------------------------ | -| `type` | String | Yes | none | Must be `s3_sigv4` | +| `type` | String | Yes | none | `s3_sigv4` | | `region` | String | Yes | none | AWS region used in the SigV4 credential scope | | `access_key_id` | String | No | `access_key_id` | Default-store secret reference for the AWS access key ID | | `secret_access_key` | String | No | `secret_access_key` | Default-store secret reference for the AWS secret access key | @@ -1701,9 +1707,9 @@ secret_access_key = "s3_secret_access_key" # session_token = "s3_session_token" ``` -S3 auth uses header-based AWS SigV4 with `UNSIGNED-PAYLOAD`. It is scoped to read-only asset requests and expects `url` to use the S3 host that AWS validates. Credential references resolve from `trusted_server_secrets` at startup, and request signing performs no secret-store reads. +`s3_sigv4` uses header-based AWS SigV4 with `UNSIGNED-PAYLOAD`. It is scoped to read-only asset requests and expects `url` to use the S3 host that AWS validates. Credential references resolve from `trusted_server_secrets` at startup, and request signing performs no secret-store reads. -Signing requires `url` to use `https`. A table the signer cannot use, such as an empty or malformed `region`, is refused at startup naming the route, as in `proxy.asset_routes /.image/ source.origin.auth`. +Signing requires `url` to use `https`. A `type` no crate in the build offers is refused at startup naming the signers on offer. A table the signer cannot use, such as an empty or malformed `region`, is refused at startup and by `ts config validate` naming the route, as in `proxy.asset_routes /.image/ source.origin.auth`. Effective `origin_query` precedence is auth-level `origin_query`, then enabled Image Optimizer `origin_query`, then the route default. diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index 9ca9a49788..834596068a 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -363,6 +363,7 @@ points at the functions that do the work. | `.with_demand(...)` and `.with_adserver(...)` | Optional. Registers an auction implementation that `[demand]` or `[ad-server]` can name. A crate that supplies nothing else starts from `IntegrationBuilder::implementations(...)` | | `.with_loading_cache(...)` | Optional. Offers the loading cache the crate supplies, which a feature selects by the module's name in its `[cache.]` table. An adapter takes the selections with `loading_caches_from` | | `.with_shared_settings(...)` | Optional. Runs the builder when its vendor's `[shared.]` table is present, for a vendor whose modules all read that one table. A builder the table runs is selected by no section | +| `.with_origin_signer(...)` | Optional. Offers an origin signer, which a source's `auth` table names with `type`, and declares the settings of the signer's table that hold the name of a secret | ```rust pub fn module_name() -> &'static str { @@ -827,6 +828,48 @@ in the table is declared with `.with_secret_settings(...)` at its path inside the table, as a module's own secrets are, and is resolved from the secret store when the table is present. +### Signing requests to an origin + +An origin that answers only signed reads, such as private object storage, is +signed through an `OriginSigner`, in `trusted_server_core::origin_auth`. Core +declares the interface and the signer comes from a crate outside core. A +source's `auth` table names it with `type`, core reads that and +`origin_query`, and the rest of the table is the signer's own. + +A crate supplies one by implementing `OriginSigner`, which signs the headers +of a request for a method on a final address and removes every header in the +scheme's own namespace, and offering an `OriginSignerModule` on its builder +with `.with_origin_signer(...)`. The module names the signer, lists the +settings of its table that hold the name of a secret, each with the key name +it takes when left out, and builds the signer from its table. The offer is not +a selection. The registry builds the signer where a source names it, so a +`type` nothing offers is refused at startup naming the signers on offer, and a +table the signer cannot use is refused naming the setting. The builder's +validate function checks the same tables when a deployment is validated, so a +bad table fails `ts config validate` and not the first start. + +```rust +pub static EXAMPLE_SIGNER: OriginSignerModule = OriginSignerModule { + name: "example_scheme", + secrets: &[SignerSecret { + name: "key_name", + default: Some("key_name"), + }], + build: build_signer, +}; + +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(MODULE, env!("CARGO_PKG_NAME"), no_registration, validate) + .with_origin_signer(&EXAMPLE_SIGNER) +} +``` + +The declared secret settings are looked up in the default secret store as +the settings load, at both places an `auth` table can be written, and masked +on the configuration page, so the signer is handed the secret itself and reads +no store of its own. The stock list carries `crates/shared/aws`, which offers +`s3_sigv4`. + ### Supplying a loading cache Some answers cost a call to make and are the same for many requests, such as diff --git a/docs/superpowers/specs/2026-10-14-origin-signing-design.md b/docs/superpowers/specs/2026-10-14-origin-signing-design.md new file mode 100644 index 0000000000..d60d3f73f2 --- /dev/null +++ b/docs/superpowers/specs/2026-10-14-origin-signing-design.md @@ -0,0 +1,60 @@ +# Design Spec: An origin signer is a module a source's auth table names + +**Status:** Proposed, 2026-10-14. Implemented on a working branch of the fork, with no pull request raised yet. +**Author:** 51Degrees (contributed), for Tech Lab review +**Related specs:** `2026-10-14-source-modules-design.md`, `2026-10-13-publisher-object-storage-origin-design.md`, `2026-05-19-asset-s3-auth-fastly-io-design.md` +**Related PRs:** none yet +**Issues:** IABTechLab/trusted-server#1338 +**Last updated:** 2026-10-14 + +> **Why this spec exists.** Core signs reads from the publisher's origin and from each asset route's origin with one cloud vendor's signing scheme, and its settings know that scheme's credentials, so an origin that wants another scheme has to add a branch to core. + +## 1. Problem + +`crates/trusted-server-core/src/s3_sigv4.rs` signs a read from object storage with AWS Signature Version 4, and the `auth` table of a source's `origin` settings is a typed enum with one variant, `s3_sigv4`, holding that scheme's region and credential key names. The publisher path and the asset routes sign through that one function, the deploy check knows the three credential fields by name, and the configuration page masks them because core declares them. + +So core knows one vendor's scheme. A signed read from another vendor's storage, or with another scheme, is a change to core's settings, its signing call and its secret declarations, for every deployment. + +## 2. Design + +**Core keeps an origin authentication interface.** `trusted_server_core::origin_auth` declares `OriginSigner`, which signs the headers of a request for a method on a final address, and removes every header in the scheme's own namespace so a reader's header there cannot reach the origin beside the signature. Core calls it where it signed before, in the asset route path, in its preflight ahead of the image optimizer, and in the publisher path, through the `OriginSigning` a `SourceLocation` carries. + +**A source's `auth` table names the signer.** The table keeps its shape. `type` names the signer, `origin_query` says whether the reader's query is kept on the signed request, and every other key belongs to the signer named. + +```toml +[source.origin] +url = "https://bucket.storage.example.com/site" + +[source.origin.auth] +type = "s3_sigv4" +region = "example-region-1" +access_key_id = "origin_access_key_id" +secret_access_key = "origin_secret_access_key" +# session_token = "origin_session_token" +# origin_query = "strip" +``` + +Core reads `type` and `origin_query` and refuses a table that names no signer, or one in a form no crate could offer. The rest of the table is handed to the signer as written. + +**A crate offers a signer on its builder.** `IntegrationBuilder::with_origin_signer(&MODULE)` offers an `OriginSignerModule`, being the name an `auth` table writes, the settings of its table that hold the name of a secret, each with the key name it takes when left out, and the function that builds the signer from its table. The offer is not a selection. A signer is built where a source names it, when the registry builds the sources from the settings, so a `type` nothing offers is refused at startup naming the signers on offer, and a table the signer cannot use is refused there naming the setting, as `[proxy.asset_routes /assets/ source.origin.auth] s3_sigv4 region must not be empty`. + +**The signer's secrets are looked up and masked as core's are.** The secret settings a signer declares are declared to the settings load at both places an `auth` table can be written, `source.origin.auth` and `proxy.asset_routes[].source.origin.auth`, so each is looked up in the default secret store as the settings load and masked on the configuration page. A setting with a default key name that the table leaves out is written in before the lookup, so it is looked up under that name as it was when core's settings typed it. A written setting that names no key is refused by `ts config validate` naming its path. + +**AWS signing is a shared vendor crate.** `crates/shared/aws`, the package `trusted-server-shared-aws`, holds the signer and its settings. It offers `s3_sigv4`, reading `region`, `access_key_id`, `secret_access_key` and `session_token` as before, with the same defaults, and signs with header-based authentication and `UNSIGNED-PAYLOAD` as before. The stock build lists its builder in `crates/trusted-server-modules`, so every adapter and the `ts` tool offer it, and its own deploy rules run with the stock modules'. + +What leaves core: the signer, its settings type, the credential fields in core's secret declarations, the credential check in the deploy validation, and the preflight's knowledge that the signed origin is S3. What stays: the `https` requirement of a signed source, the `405` for a method that is not a read, the reader headers a signed read never carries, and `origin_query`. + +## 3. What is not in this change + +- A source module from a vendor crate signing through a signer another crate offers. Core's `origin` source is handed the signers on offer when it is built. A vendor source that signs holds its own signer. +- The deprecated spelling `s3_sig_v4`, which core's settings read as an alias. A `type` is matched as written, and a document carrying the old spelling is refused naming the names on offer. + +## 4. Tests + +- An `auth` table reads its `type` and `origin_query` and keeps the rest for the signer, serializes as written, and refuses a missing or malformed `type` naming the setting. +- A `type` nothing offers is refused with the names on offer, and with none on offer says so. A table the signer cannot use is refused naming the setting, for the publisher's source and for an asset route's, naming the route. +- Two builders offering one signer name are refused. +- A signer's secret settings are declared at both `auth` paths, a left out setting with a default is written in before the lookup, a written setting naming no key fails deploy validation naming its path, and a loaded configuration masks every one on the configuration page and serves none. +- The asset route path signs the final address and `Host`, strips the transform query before signing, removes a reader's header in the signer's namespace, preflights a signed origin with a `HEAD` before the image optimizer, signing each request as its own method, and returns the origin's own error. +- The publisher path signs exactly the address and `Host` it sends, as a `GET` or a `HEAD`, carries none of the reader's private headers, strips the query when the table says so, and refuses a method that is not a read. +- The `s3_sigv4` signer signs the published example to the published signature, signs a session token, refuses a request without a `Host`, and canonicalizes paths and queries as before. Through the registry, it signs a publisher source and an asset route, refuses a table it cannot use at startup and at deploy validation, is refused where no build offers it, and its credentials are looked up at both `auth` paths and masked. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 54e9a6996d..c9401e3250 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -85,10 +85,12 @@ url = "https://origin.example.com" # with its index file. The reader's address is unchanged. # directory_index = "index.html" # Optional: sign every origin request, for an origin such as private object -# storage. The same table as an asset route's. Only GET and HEAD are signed, -# other methods are refused, and the reader's cookies, Authorization, Referer, -# conditional validators and Range are not sent. url must use https. Keep this -# table last in [source.origin]. +# storage. The same table as an asset route's. type names an origin signer a +# crate in the build offers, s3_sigv4 in a stock build, and the rest of the +# table is that signer's. Only GET and HEAD are signed, other methods are +# refused, and the reader's cookies, Authorization, Referer, conditional +# validators and Range are not sent. url must use https. Keep this table last +# in [source.origin]. # [source.origin.auth] # type = "s3_sigv4" # region = "example-region-1" @@ -447,12 +449,12 @@ pull_sync_concurrency = 3 # [proxy.asset_routes.source.origin] # url = "https://assets.example.com" # -# Optional S3 SigV4 auth for a private origin: +# Optional signing for a private origin. type names an origin signer a crate +# in the build offers, s3_sigv4 in a stock build: # [proxy.asset_routes.source.origin.auth] # type = "s3_sigv4" # region = "us-east-1" # origin_query = "strip" -# secret_store = "s3-auth" # access_key_id = "access_key_id" # secret_access_key = "secret_access_key" # From 94d53bc46436eebe870f7f7fdc8fcee02128f1b4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 14 Oct 2026 23:57:04 +0100 Subject: [PATCH 2/7] The signer crate's tests read a registry refusal by matching on it A registry carries no Debug, so a test that expects a refusal from building one matches on the result rather than calling expect_err, which the check of the signer crate's tests refused. --- crates/shared/aws/src/lib.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/shared/aws/src/lib.rs b/crates/shared/aws/src/lib.rs index 514fdf8f9c..776ec49aea 100644 --- a/crates/shared/aws/src/lib.rs +++ b/crates/shared/aws/src/lib.rs @@ -312,6 +312,18 @@ mod tests { IntegrationRegistry::with_registrations(settings, &[builder()]) } + /// The refusal a registry build ended in, or a panic saying `expected` + /// when it built. + fn refused( + result: Result>, + expected: &str, + ) -> Report { + match result { + Err(err) => err, + Ok(_) => panic!("{expected}"), + } + } + #[test] fn the_publisher_source_signs_through_the_signer_the_auth_names() { let settings = settings_signing_the_origin(auth(Some(OriginQueryPolicy::Strip))); @@ -458,7 +470,7 @@ mod tests { ), ] { let settings = settings_signing_the_origin(table.clone()); - let err = registry(&settings).expect_err("should refuse the table at startup"); + let err = refused(registry(&settings), "should refuse the table at startup"); assert!( format!("{err:?}").contains(at_startup), "{table:?} should be refused saying `{at_startup}`: {err:?}" @@ -483,7 +495,7 @@ mod tests { ); settings.proxy.asset_routes.push(route); - let err = registry(&settings).expect_err("should refuse the route's table"); + let err = refused(registry(&settings), "should refuse the route's table"); assert!( format!("{err:?}").contains( "[proxy.asset_routes /assets/ source.origin.auth] s3_sigv4 region must not be empty" @@ -496,8 +508,10 @@ mod tests { fn the_signer_is_refused_where_the_build_does_not_offer_it() { let settings = settings_signing_the_origin(auth(None)); - let err = IntegrationRegistry::with_registrations(&settings, &[]) - .expect_err("should refuse a signer nothing offers"); + let err = refused( + IntegrationRegistry::with_registrations(&settings, &[]), + "should refuse a signer nothing offers", + ); assert!( format!("{err:?}").contains( "[source.origin.auth] type `s3_sigv4` names an origin signer this deployment does not offer" From 4079e3a0738b8e777e6f7af36a54bf97d180f5d6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 15 Oct 2026 01:14:05 +0100 Subject: [PATCH 3/7] The lock lists the AWS crate after the merge --- Cargo.lock | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 32129cc2cf..f753e8c6a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6627,6 +6627,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", "trusted-server-testing-iabtl", @@ -6675,6 +6676,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", "trusted-server-testing-testlight", @@ -6727,6 +6729,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", "trusted-server-testing-iabtl", @@ -6777,6 +6780,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", "trusted-server-testing-testlight", @@ -6999,6 +7003,7 @@ dependencies = [ "trusted-server-identity-lockr", "trusted-server-loading-cache-51degrees", "trusted-server-middleware-51degrees", + "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", "trusted-server-testing-testlight", @@ -7333,6 +7338,25 @@ dependencies = [ "validator", ] +[[package]] +name = "trusted-server-shared-aws" +version = "0.1.0" +dependencies = [ + "chrono", + "edgezero-core", + "error-stack", + "futures", + "hex", + "hmac", + "http", + "log", + "serde", + "serde_json", + "sha2 0.10.9", + "trusted-server-core", + "url", +] + [[package]] name = "trusted-server-tag-google-tag" version = "0.1.0" From dd5a4b6568f6c04d4d94052f85b8cbc524579575 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 15 Oct 2026 01:57:02 +0100 Subject: [PATCH 4/7] Core checks each auth table with its signer at deploy validation Deploy validation builds every auth table that names an offered signer, as the settings load does, so a table the signer cannot use fails validation for any signer. The AWS builder then needs no validator of its own and offers the signer as an implementation, with no page module, which the shipped-modules test expects of a builder without a section. The S3 secret paths leave core's static list, because the signers the builders offer declare them, and the shared README and the AWS builder's comment name the adapters' module lists in place of the deleted modules crate. --- crates/shared/README.md | 4 +- crates/shared/aws/src/lib.rs | 39 ++------------- crates/trusted-server-core/src/config.rs | 50 ++++++++++++------- crates/trusted-server-core/src/origin_auth.rs | 29 +++++++++++ 4 files changed, 69 insertions(+), 53 deletions(-) diff --git a/crates/shared/README.md b/crates/shared/README.md index 0e80b7bd64..3f62cc63e7 100644 --- a/crates/shared/README.md +++ b/crates/shared/README.md @@ -9,8 +9,8 @@ core declares for that thing. - `crates/shared/aws` (`trusted-server-shared-aws`) offers the origin signer `s3_sigv4`, which a source's `auth` table names to sign a read from S3 with AWS Signature Version 4. Core's `origin_auth` module declares the signer - interface, the `OriginSigner` trait, and the stock build offers this signer - through `crates/trusted-server-modules`. + interface, the `OriginSigner` trait, and each adapter lists the crate's + builder among the modules it links. A vendor whose crates share settings, such as an endpoint and a license key, writes them in one `[shared.]` table, which its builder reads with diff --git a/crates/shared/aws/src/lib.rs b/crates/shared/aws/src/lib.rs index 776ec49aea..e0210a1c6a 100644 --- a/crates/shared/aws/src/lib.rs +++ b/crates/shared/aws/src/lib.rs @@ -50,10 +50,9 @@ use http::{HeaderMap, Method}; use serde::Deserialize; use serde_json::{Map, Value}; use trusted_server_core::error::TrustedServerError; -use trusted_server_core::integrations::{IntegrationBuilder, IntegrationRegistration}; +use trusted_server_core::integrations::IntegrationBuilder; use trusted_server_core::origin_auth::{OriginSigner, OriginSignerModule, SignerSecret}; use trusted_server_core::redacted::Redacted; -use trusted_server_core::settings::Settings; use url::Url; /// The id of this crate's builder, from the crate's folder. @@ -87,43 +86,15 @@ pub static S3_SIGV4: OriginSignerModule = OriginSignerModule { }; /// The builder a deployment hands to an adapter, which offers the signer -/// whatever the settings say. A stock build lists it in -/// `trusted-server-modules`. +/// whatever the settings say. It supplies no page module, and core checks +/// each `auth` table naming the signer when a deployment is validated. Each +/// adapter lists it among the modules it links. #[must_use] pub fn builder() -> IntegrationBuilder { - IntegrationBuilder::new(MODULE, env!("CARGO_PKG_NAME"), no_registration, validate) + IntegrationBuilder::implementations(MODULE, env!("CARGO_PKG_NAME")) .with_origin_signer(&S3_SIGV4) } -/// The builder registers no page integration, because no section selects it. -fn no_registration( - _settings: &Settings, -) -> Result, Report> { - Ok(None) -} - -/// Checks every `auth` table naming the signer, for a deployment being -/// validated, so a table the signer cannot use fails `ts config validate` -/// and not the deployment's first start. -fn validate(settings: &Settings) -> Result> { - if let Some(auth) = settings.origin_source().and_then(|origin| origin.auth) - && auth.signer == SIGNER - { - S3SigV4Settings::read(&auth.settings, "source.origin.auth")?; - } - for route in &settings.proxy.asset_routes { - if let Some(auth) = route.origin_source().and_then(|origin| origin.auth) - && auth.signer == SIGNER - { - S3SigV4Settings::read( - &auth.settings, - &format!("{}.origin.auth", route.source_setting()), - )?; - } - } - Ok(false) -} - fn build_signer( settings: &Map, setting: &str, diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index b7424ed0b8..ca8ec3c0d9 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -496,12 +496,14 @@ fn validate_secret_key_references( } } - // An origin signer's secret settings are known through the builders that - // offer the signer, as they are when the settings load. + // An origin signer's secret settings, and the tables it can use, are + // known through the builders that offer the signer, as they are when the + // settings load. let builders: Vec = crate::integrations::all_builders(extra_integrations).collect(); let signers = crate::origin_auth::OriginSigners::offered_by(&builders)?; crate::origin_auth::check_key_references(settings, &signers)?; + crate::origin_auth::check_tables(settings, &signers)?; Ok(()) } @@ -956,21 +958,6 @@ formats = [{ width = 300, height = 250 }] ("ec.partners[*].api_token".to_owned(), true), ("ec.partners[*].ts_pull_token".to_owned(), true), ("trusted_client_ip.shared_secret".to_owned(), false), - ( - "proxy.asset_routes[*].source.origin.auth.access_key_id".to_owned(), - true - ), - ( - "proxy.asset_routes[*].source.origin.auth.secret_access_key".to_owned(), - true, - ), - ( - "proxy.asset_routes[*].source.origin.auth.session_token".to_owned(), - true - ), - ("source.origin.auth.access_key_id".to_owned(), true), - ("source.origin.auth.secret_access_key".to_owned(), true), - ("source.origin.auth.session_token".to_owned(), true), ], "should expose the native EdgeZero secret metadata contract" ); @@ -1281,6 +1268,35 @@ passphrase = "production-secret-key-32-bytes-min" .expect("the key name is the signer's business, and no builder offers it here"); } + #[test] + fn deploy_validation_refuses_an_auth_table_its_signer_cannot_use() { + let mut publisher = valid_settings(); + edit_origin(&mut publisher, |origin| { + origin.url = "https://storage.example.com/bucket/site".to_string(); + origin.auth = Some(fixture::auth(" ")); + }); + let mut route = + crate::settings::ProxyAssetRoute::new("/assets/", "https://assets.example.com"); + crate::test_support::tests::set_route_auth(&mut route, Some(fixture::auth(" "))); + let mut routed = valid_settings(); + routed.proxy.asset_routes.push(route); + + for (settings, table) in [ + (publisher, "source.origin.auth"), + (routed, "proxy.asset_routes /assets/ source.origin.auth"), + ] { + let err = validate_settings_for_deploy_with(&settings, &[fixture::builder()]) + .expect_err("should refuse a table the signer cannot use"); + assert!( + format!("{err:?}").contains(&format!( + "[{table}] {} realm must not be empty", + fixture::SIGNER + )), + "should name `{table}`: {err:?}" + ); + } + } + #[test] fn documented_source_origin_object_storage_settings_validate_when_uncommented() { let toml = uncomment_block( diff --git a/crates/trusted-server-core/src/origin_auth.rs b/crates/trusted-server-core/src/origin_auth.rs index a4457dc836..ed9b31e1e3 100644 --- a/crates/trusted-server-core/src/origin_auth.rs +++ b/crates/trusted-server-core/src/origin_auth.rs @@ -469,6 +469,35 @@ pub(crate) fn check_key_references( } } +/// Builds every `auth` table that names a signer `signers` offers, for a +/// check that runs while the settings still hold key names, so a table the +/// signer cannot use fails deploy validation and not the deployment's first +/// start. A table naming a signer nothing offers is left to the build that +/// offers it. +/// +/// # Errors +/// +/// The signer's refusal, naming the table as the settings load does. +pub(crate) fn check_tables( + settings: &Settings, + signers: &OriginSigners, +) -> Result<(), Report> { + let publisher = settings + .origin_source() + .and_then(|origin| origin.auth) + .map(|auth| (auth, "source.origin.auth".to_owned())); + let routes = settings.proxy.asset_routes.iter().filter_map(|route| { + let auth = route.origin_source()?.auth?; + Some((auth, format!("{}.origin.auth", route.source_setting()))) + }); + for (auth, setting) in publisher.into_iter().chain(routes) { + if signers.get(&auth.signer).is_some() { + signers.build(&auth, &setting)?; + } + } + Ok(()) +} + /// A signer for core's tests, which cannot link the crates that supply the /// real ones. /// From 5c43f6f9a8fa2ab714427e37c308c2b9f351937b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 15 Oct 2026 17:27:13 +0100 Subject: [PATCH 5/7] The origin signing design states the gap in deploy validation and what changes for a deployment Review round 1 found that the design named a modules crate that no longer exists and deploy rules the AWS builder does not carry, and that a type nothing offers passes ts config validate and ts config push and only stops the adapter at startup. The design now says so, calls the removal of s3_sig_v4 breaking, and leaves closing the gap as an open question. It also gives the refusals by stage, the answers when signing fails, the rollout, the adapters that run each path, the settings that change and the tests that are missing or cannot fail, and records the review round. The changelog entry, the integration guide and the configuration guide drop the same stale sentences: the modules crate, the builder's validate function, and a blank session token read as no token. --- CHANGELOG.md | 2 +- docs/guide/configuration.md | 6 +- docs/guide/integration-guide.md | 7 +- .../specs/2026-10-14-origin-signing-design.md | 70 +++++++++++++++---- 4 files changed, 66 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efec091912..d58fc5d2ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- **Breaking for a build that links its own crates:** a source's `auth` table names an origin signer a crate offers. Core keeps the interface, `OriginSigner` in `trusted_server_core::origin_auth`, and signs the publisher's origin requests and each asset route's through it where it signed before. `type` names the signer, `origin_query` is read by core, and the rest of the table is the signer's. The `s3_sigv4` signer moves to `crates/shared/aws`, keeping its settings, defaults and secret-store key references, and a stock build still offers it through `crates/trusted-server-modules`, so a deployment's settings read as before. A build that composes its own modules adds `trusted_server_shared_aws::builder()`, or a `type` of `s3_sigv4` is refused at startup naming the signers on offer. A table the signer cannot use is refused at startup and by `ts config validate` naming the setting. The deprecated spelling `s3_sig_v4` is no longer read. **For crates outside core**, `AssetOriginAuth` and `S3SigV4AuthConfig` are replaced by `OriginAuth`, `SourceLocation::auth` gives an `OriginSigning`, `OriginSettings::location` and `OriginSource::new` take the signers on offer and the setting's name, and a crate offers a signer with `IntegrationBuilder::with_origin_signer`. +- **Breaking for a build that links its own crates:** a source's `auth` table names an origin signer a crate offers. Core keeps the interface, `OriginSigner` in `trusted_server_core::origin_auth`, and signs the publisher's origin requests and each asset route's through it where it signed before. `type` names the signer, `origin_query` is read by core, and the rest of the table is the signer's. The `s3_sigv4` signer moves to `crates/shared/aws`, keeping its settings, defaults and secret-store key references, and every adapter and the `ts` tool link the crate's builder, so a stock build still offers it. A build that composes its own modules adds `trusted_server_shared_aws::builder()`, or a `type` of `s3_sigv4` is refused at startup naming the signers on offer. A table the signer cannot use is refused at startup and by `ts config validate` naming the setting. The deprecated spelling `s3_sig_v4` is no longer read. **For crates outside core**, `AssetOriginAuth` and `S3SigV4AuthConfig` are replaced by `OriginAuth`, `SourceLocation::auth` gives an `OriginSigning`, `OriginSettings::location` and `OriginSource::new` take the signers on offer and the setting's name, and a crate offers a signer with `IntegrationBuilder::with_origin_signer`. - **Breaking for crates outside core:** an adapter answers how it names a backend and bounds its transport timers through `PlatformBackend` alone, which gains `auction_backend_budget` and `transport_timeout_bucket_count` with defaults, and describes itself to auction plan validation through an `AuctionTargetDescriptor` it builds from its name, its backend implementation and its `AuctionTargetCapabilities`. `AuctionPlan::validate_for_target` takes that descriptor, and `BackendNamingPolicy`, `AuctionTargetId`, `PredictedBackend`, `BackendNamingError` and `PlatformBackend::naming_policy` are gone from core, because core names no hosting platform. Each adapter carries its own naming policy beside the code that registers its backends, and the names a deployment sees are unchanged. - The adapter declares the headers its platform reads a shared cache policy from. Core reads `Surrogate-Control` and `CDN-Cache-Control` for every platform, and the Fastly and Cloudflare adapters declare `fastly-surrogate-control` and `cloudflare-cdn-cache-control` as they build their state, so each is still stripped from every response core makes private. Responses carry the headers they carried. **For crates outside core**, `EdgeCacheHeader::CloudflareCdnCacheControl` is `EdgeCacheHeader::Platform(name)`, `EDGE_CACHE_HEADER_NAMES` is `cache_policy::edge_cache_header_names()`, the two platform header constants leave core, and an adapter declares its platform's headers with `platform::declare_platform_cache_headers`. - Auction telemetry is an analytics module from a crate of its own. `[analytics]` is read as every module type's section is, and the Tinybird module, its settings and its checks live in `crates/analytics/tinybird`, offered by `crates/trusted-server-modules` to every adapter, where the Fastly adapter alone built it. The settings a deployment writes do not change. A table missing its host or token, a key the table does not know and a reserved setting turned on are refused when the deployment starts and by `ts config validate`, naming the table, where they were refused as the settings loaded. A `[tinybird]` table at the top level is refused as the section of a module type nothing supplies. **For crates outside core**, a registration declares the sink it supplies with `IntegrationRegistrationBuilder::with_analytics_module`, and `IntegrationRegistry::analytics_module` gives the one `[analytics] module` selected. `Settings::analytics_module`, `AnalyticsConfig` and `TinybirdSettings` leave core. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index d290ae75cf..2c9ff409af 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -91,8 +91,10 @@ publisher, trusted-client-IP, EC, Tinybird, DataDome, and S3 fields: Their values belong in the logical `trusted_server_secrets` store and are resolved only while an instance builds runtime settings. An adapter can map the logical ID to a different physical name. For example, Fastly commonly maps -`trusted_server_secrets` to physical store `ts_secrets`. A blank -`session_token` in either signing table is read as no token. +`trusted_server_secrets` to physical store `ts_secrets`. A `session_token` +that is empty or only spaces in either signing table is refused by +`ts config validate`, naming its path, so leave the key out when there is no +token. Two accepted secret-shaped fields are deliberately different: diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index b5f580423b..2733e88c00 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -887,9 +887,10 @@ settings of its table that hold the name of a secret, each with the key name it takes when left out, and builds the signer from its table. The offer is not a selection. The registry builds the signer where a source names it, so a `type` nothing offers is refused at startup naming the signers on offer, and a -table the signer cannot use is refused naming the setting. The builder's -validate function checks the same tables when a deployment is validated, so a -bad table fails `ts config validate` and not the first start. +table the signer cannot use is refused naming the setting. Core checks the +same tables when a deployment is validated, by building each signer a builder +offers from the tables that name it, so a bad table fails `ts config validate` +and not the first start, provided the validating build offers that signer. ```rust pub static EXAMPLE_SIGNER: OriginSignerModule = OriginSignerModule { diff --git a/docs/superpowers/specs/2026-10-14-origin-signing-design.md b/docs/superpowers/specs/2026-10-14-origin-signing-design.md index d60d3f73f2..081fc573f5 100644 --- a/docs/superpowers/specs/2026-10-14-origin-signing-design.md +++ b/docs/superpowers/specs/2026-10-14-origin-signing-design.md @@ -1,11 +1,11 @@ # Design Spec: An origin signer is a module a source's auth table names -**Status:** Proposed, 2026-10-14. Implemented on a working branch of the fork, with no pull request raised yet. +**Status:** Proposed, 2026-10-14. Raised as draft pull request jwrosewell/trusted-server#43, and as draft pull request IABTechLab/trusted-server#1391, for Tech Lab review. **Author:** 51Degrees (contributed), for Tech Lab review -**Related specs:** `2026-10-14-source-modules-design.md`, `2026-10-13-publisher-object-storage-origin-design.md`, `2026-05-19-asset-s3-auth-fastly-io-design.md` -**Related PRs:** none yet +**Related specs:** `2026-10-14-source-modules-design.md`, `2026-10-13-publisher-object-storage-origin-design.md`, `2026-05-19-asset-s3-auth-fastly-io-design.md` and `2026-10-14-no-modules-crate-design.md`, all in IABTechLab/trusted-server#1084, which holds every design document of the stack +**Related PRs:** jwrosewell/trusted-server#43, IABTechLab/trusted-server#1391 **Issues:** IABTechLab/trusted-server#1338 -**Last updated:** 2026-10-14 +**Last updated:** 2026-10-15 > **Why this spec exists.** Core signs reads from the publisher's origin and from each asset route's origin with one cloud vendor's signing scheme, and its settings know that scheme's credentials, so an origin that wants another scheme has to add a branch to core. @@ -17,7 +17,9 @@ So core knows one vendor's scheme. A signed read from another vendor's storage, ## 2. Design -**Core keeps an origin authentication interface.** `trusted_server_core::origin_auth` declares `OriginSigner`, which signs the headers of a request for a method on a final address, and removes every header in the scheme's own namespace so a reader's header there cannot reach the origin beside the signature. Core calls it where it signed before, in the asset route path, in its preflight ahead of the image optimizer, and in the publisher path, through the `OriginSigning` a `SourceLocation` carries. +**Core keeps an origin authentication interface.** `trusted_server_core::origin_auth` declares `OriginSigner`, which signs the headers of a request for a method on a final address, and removes every header in the scheme's own namespace so a reader's header there cannot reach the origin beside the signature. Core calls it where it signed before, in the asset route path, in the preflight that path runs ahead of the image optimizer, and in the publisher path, through the `OriginSigning` a `SourceLocation` carries. The publisher path is core's own, so it signs on all four adapters. Asset routes and the preflight run on the Fastly adapter alone, because only that adapter calls `handle_asset_proxy_request`. + +Three things differ from what core did before. First, the `HEAD` preflight, which asked an S3 origin for the object before the image optimizer did, now runs for any signed asset route that uses the optimizer, whatever its signer. Second, removing the headers in the signer's namespace is each signer's duty, so core calls `remove_namespace_headers` before `sign` on the publisher path and the asset route path, and does not check afterwards that none is left. Third, the asset route path's call to `remove_namespace_headers` is new, because that call was made on the publisher path alone. The asset route path starts its outbound headers from eleven named request headers (`ASSET_PROXY_FORWARD_HEADERS` in `crates/trusted-server-core/src/proxy.rs`) and none of them is in the `x-amz-` namespace, so for `s3_sigv4` the call removes nothing. **A source's `auth` table names the signer.** The table keeps its shape. `type` names the signer, `origin_query` says whether the reader's query is kept on the signed request, and every other key belongs to the signer named. @@ -34,27 +36,69 @@ secret_access_key = "origin_secret_access_key" # origin_query = "strip" ``` -Core reads `type` and `origin_query` and refuses a table that names no signer, or one in a form no crate could offer. The rest of the table is handed to the signer as written. +Core reads `type` and `origin_query` and hands the rest of the table to the signer as written. It trims `type` and refuses a table that names no signer, or one whose `type` could not be a crate's module name, which is parts joined by `.`, each of lower case letters, digits, `_` or `-`. `origin_query` is `preserve` or `strip`. On the publisher's source, leaving it out keeps the reader's query. On an asset route the value in the `auth` table wins, then the `origin_query` of an enabled image optimizer, then the route's default, which is `strip` when the image optimizer is enabled and `preserve` otherwise. **A crate offers a signer on its builder.** `IntegrationBuilder::with_origin_signer(&MODULE)` offers an `OriginSignerModule`, being the name an `auth` table writes, the settings of its table that hold the name of a secret, each with the key name it takes when left out, and the function that builds the signer from its table. The offer is not a selection. A signer is built where a source names it, when the registry builds the sources from the settings, so a `type` nothing offers is refused at startup naming the signers on offer, and a table the signer cannot use is refused there naming the setting, as `[proxy.asset_routes /assets/ source.origin.auth] s3_sigv4 region must not be empty`. -**The signer's secrets are looked up and masked as core's are.** The secret settings a signer declares are declared to the settings load at both places an `auth` table can be written, `source.origin.auth` and `proxy.asset_routes[].source.origin.auth`, so each is looked up in the default secret store as the settings load and masked on the configuration page. A setting with a default key name that the table leaves out is written in before the lookup, so it is looked up under that name as it was when core's settings typed it. A written setting that names no key is refused by `ts config validate` naming its path. +**The signer's secrets are looked up as core's are, and masked through the settings load's record.** The secret settings a signer declares are declared to the settings load at both places an `auth` table can be written, `source.origin.auth` and `proxy.asset_routes[].source.origin.auth`, so each is looked up in the default secret store as the settings load. A setting with a default key name that the table leaves out is written in before the lookup, so it is looked up under that name as it was when core's settings typed it. A written setting that is empty, only spaces or not text is refused by `ts config validate` naming its path. On a running adapter, the configuration page masks these values through the record the settings load keeps of the secrets it looked up, and through nothing else. Core's own secret declarations no longer name a signer's settings, and the values sit as plain text in the signer's table and no longer in `Redacted` fields, which mark themselves for the page. Every adapter loads its settings with the builders it links, so each masks the settings of the signers it links. -**AWS signing is a shared vendor crate.** `crates/shared/aws`, the package `trusted-server-shared-aws`, holds the signer and its settings. It offers `s3_sigv4`, reading `region`, `access_key_id`, `secret_access_key` and `session_token` as before, with the same defaults, and signs with header-based authentication and `UNSIGNED-PAYLOAD` as before. The stock build lists its builder in `crates/trusted-server-modules`, so every adapter and the `ts` tool offer it, and its own deploy rules run with the stock modules'. +**AWS signing is a shared vendor crate.** `crates/shared/aws`, the package `trusted-server-shared-aws`, holds the signer and its settings. It offers `s3_sigv4`, reading `region`, `access_key_id`, `secret_access_key` and `session_token` as before, with the same defaults, and signs with header-based authentication and `UNSIGNED-PAYLOAD` as before. The changes to the settings a deployment writes are listed under "What changes for settings that name `s3_sigv4`". The crate's builder, `trusted_server_shared_aws::builder()`, supplies no page module and carries no deploy rules of its own. Each adapter lists the builder in `shipped_module_builders()` in its `app.rs`, and the `ts` tool lists it in `linked_builders()` in `crates/trusted-server-cli/src/app_config.rs`, which is how `2026-10-14-no-modules-crate-design.md` has each build list the modules it links. Core checks an `auth` table against the signer it names at deploy validation, for every signer a builder offers. What leaves core: the signer, its settings type, the credential fields in core's secret declarations, the credential check in the deploy validation, and the preflight's knowledge that the signed origin is S3. What stays: the `https` requirement of a signed source, the `405` for a method that is not a read, the reader headers a signed read never carries, and `origin_query`. +**Nothing consent related changes.** This change touches no permission, consent or identifier code. The ten reader headers that a signed publisher read never carries, being `Cookie`, `Authorization`, `Proxy-Authorization`, `Referer`, `Range` and the five conditional headers, are the same ten as before, and the publisher path's signing step removes no other reader header, so the rest reach the origin as they did before, apart from any the signer removes from its own namespace. The asset route path still forwards only its eleven named headers, which include no cookie. + +**When each mistake in an `auth` table is refused.** Three stages refuse a mistake in an `auth` table. + +1. Reading the settings, which `ts config validate`, `ts config push` and an adapter's start all do first, refuses a table with no `type`, a `type` that is empty or could not be a module name, and a signed `url` that is not `https`, each naming the setting. That stage passes any other `type`, whether a builder offers it or not. +2. The deploy check, which `ts config validate` and `ts config push` run on the settings that were read, looks at each table whose `type` is a signer that the builders linked into the `ts` tool offer. It refuses a secret setting that is empty, only spaces or not text, naming its path, and it builds the signer from the table, so whatever the signer cannot use is refused naming the setting. It leaves alone a table whose `type` none of those builders offers. +3. An adapter's start builds the adapter's state, and with it the signer for every table, so that stage refuses what the signer cannot use and also a `type` that no builder the adapter links offers, naming the signers on offer. + +**The gap.** A `type` that could be a module name but that no builder offers passes the first two stages and is refused only by the third. A misspelled `type`, the removed spelling `s3_sig_v4`, and the name of a signer that the `ts` tool does not link all pass `ts config validate` and `ts config push`, are stored, and then stop the adapter when it builds its state. That adapter answers each request with its startup error response, which is a `500` with the body "An internal error occurred" on Fastly, Axum and Cloudflare and a generic `503` on Spin, and it logs the full error. Before this change core refused any `type` other than `s3_sigv4` and `s3_sig_v4` while the settings were read, so the gap is new. The skip is deliberate, and the test `deploy_validation_leaves_a_signer_s_settings_alone_without_its_builder` in `crates/trusted-server-core/src/config.rs` pins it for a table whose signer's builder is not among those offered. + +**Open question for the reviewer.** Should deploy validation close this gap or keep it? Closing the gap means refusing a `type` that no builder the `ts` tool links offers, so a misspelled `type` and the removed `s3_sig_v4` fail `ts config validate`. A deployment whose adapter links a signer of its own could then no longer push with the stock `ts` tool, which does not link that crate. The integration guide already describes the same limit for a module a deployment adds. Keeping the gap leaves a signer of the deployment's own pushable with the stock `ts` tool, and leaves a mistaken `type` to stop the adapter when it builds its state. + +**When signing fails for a request.** A signer that cannot sign a request returns its error, and core does not send the request it could not sign. The asset route path, its preflight and the publisher path hand that error to the adapter, which answers with the status the error's kind gives, as it does for any other error from the same handler. The `s3_sigv4` signer returns `502 Bad Gateway` when the request has no `Host` header and `400 Bad Request` when a header value is not valid text, with the bodies "An internal error occurred" and "Invalid header value". The asset route path marks that answer private and not stored. + +**What changes for settings that name `s3_sigv4`.** Three things change for a `trusted-server.toml` document that writes an `s3_sigv4` table: + +1. The spelling `s3_sig_v4` is no longer read, and the removal is a breaking change. Before this change core read it as an alias and wrote `s3_sigv4` into the pushed configuration. A document that still carries it now passes the first two stages above and stops the adapter, as the gap describes. +2. A blank `session_token` is refused. Before this change, reading the settings trimmed a blank `session_token` to none, so it passed `ts config validate` and the signer used no token. `ts config validate` now refuses a written `session_token` that is empty or only spaces, naming its path, as it already refused a blank `access_key_id` or `secret_access_key`. The signer still reads a blank value that it is handed as no token. +3. A deprecated `secret_store` key is kept. Before this change a `secret_store` key was accepted and left out of the pushed configuration. A table now keeps every key it is written with, so `secret_store` is pushed as written, and the signer ignores it and logs a warning each time it builds. + +**Rollout.** A build that composes its own list of modules adds `trusted_server_shared_aws::builder()` to the list. Without it a `type` of `s3_sigv4` stops that build's adapter at startup, naming the signers on offer, and a `ts` tool built the same way skips every `s3_sigv4` table at deploy validation. The keys of an `s3_sigv4` table and what they mean are unchanged, so the new build reads a configuration the old tool pushed, the old build reads one the new tool pushed, and the tool and the adapter build can be upgraded in either order. The exception is the spelling `s3_sig_v4`. The old tool stored it as `s3_sigv4` and the new tool stores it as written, so a document that carries it is corrected to `s3_sigv4` before it is pushed with the new tool. + ## 3. What is not in this change - A source module from a vendor crate signing through a signer another crate offers. Core's `origin` source is handed the signers on offer when it is built. A vendor source that signs holds its own signer. -- The deprecated spelling `s3_sig_v4`, which core's settings read as an alias. A `type` is matched as written, and a document carrying the old spelling is refused naming the names on offer. +- Refusing a `type` that no builder offers at deploy validation, which the open question above leaves to the reviewer. ## 4. Tests -- An `auth` table reads its `type` and `origin_query` and keeps the rest for the signer, serializes as written, and refuses a missing or malformed `type` naming the setting. +- An `auth` table reads its `type` and `origin_query` and keeps the rest for the signer, trims `type`, serializes as written, and refuses a missing or malformed `type` naming the setting. - A `type` nothing offers is refused with the names on offer, and with none on offer says so. A table the signer cannot use is refused naming the setting, for the publisher's source and for an asset route's, naming the route. - Two builders offering one signer name are refused. - A signer's secret settings are declared at both `auth` paths, a left out setting with a default is written in before the lookup, a written setting naming no key fails deploy validation naming its path, and a loaded configuration masks every one on the configuration page and serves none. -- The asset route path signs the final address and `Host`, strips the transform query before signing, removes a reader's header in the signer's namespace, preflights a signed origin with a `HEAD` before the image optimizer, signing each request as its own method, and returns the origin's own error. -- The publisher path signs exactly the address and `Host` it sends, as a `GET` or a `HEAD`, carries none of the reader's private headers, strips the query when the table says so, and refuses a method that is not a read. -- The `s3_sigv4` signer signs the published example to the published signature, signs a session token, refuses a request without a `Host`, and canonicalizes paths and queries as before. Through the registry, it signs a publisher source and an asset route, refuses a table it cannot use at startup and at deploy validation, is refused where no build offers it, and its credentials are looked up at both `auth` paths and masked. +- Deploy validation builds the table of a signer on offer, so a table the signer cannot use fails it naming the setting, and it leaves a table alone when its signer's builder is not among those offered. +- The asset route path signs the final address and `Host`, strips the transform query before signing, preflights a signed origin with a `HEAD` before the image optimizer, signing each request as its own method, and returns the origin's own error. +- The publisher path signs exactly the address and `Host` it sends, as a `GET` or a `HEAD`, carries none of the reader's private headers or of its headers in the signer's namespace, strips the query when the table says so, and refuses a method that is not a read. +- The `s3_sigv4` signer signs the published example to the published signature, signs a session token, refuses a request without a `Host`, reads a blank `session_token` that it is handed as no token, discards a deprecated `secret_store`, and canonicalizes paths and queries as before. Through the registry, it signs a publisher source and an asset route, refuses a table it cannot use at startup and at deploy validation, is refused where no build offers it, and its credentials are looked up at both `auth` paths and masked. +- Each adapter's test `the_linked_modules_are_offered_in_hook_order` lists `shared.aws` after the other builders that adapter links, for Fastly, Axum, Cloudflare and Spin. + +Four cases have no test that can fail: + +1. The asset route test that sends a reader's header in the signer's namespace cannot fail, because the route builds its outbound headers from eleven named request headers and that header is never among them. The test passes with the route's call to `remove_namespace_headers` removed, and no test gives that call a header it would remove. +2. The test `deploy_validation_leaves_a_signer_s_settings_alone_without_its_builder` in `crates/trusted-server-core/src/config.rs` shows deploy validation leaving a table alone when no builder offers its signer, with the core fixture signer and a blank key name. No test sends deploy validation a `type` that names no signer at all, such as a misspelling or `s3_sig_v4`, or shows the same settings then refused when the adapter builds its state. +3. No test gives an `s3_sigv4` table an empty `session_token`. Its refusal is tested only through the core fixture signer's `token_name`, a setting declared the same way. +4. No test in the `ts` tool's crate names the AWS builder or an `s3_sigv4` table, so nothing fails if the builder leaves the list in `linked_builders()`, and `ts config validate` would then skip every `s3_sigv4` table. + +## Review log + +1. **Round 1, 2026-10-15.** A review in a fresh session, against `split/48-aws-signing` at `a3c17125b`. Issues found, two of them blocking. The section on AWS signing named a modules crate that no longer exists and deploy rules the builder does not carry, and the spec left out that a `type` nothing offers passes `ts config validate` and `ts config push` and only stops the adapter at startup, while it listed the removed `s3_sig_v4` spelling under what is not in the change. This revision corrects both, states which stage refuses each mistake, what a failed signature answers, the rollout, the adapters that run each path, the settings that change and the tests that are missing or cannot fail, leaves closing the gap as an open question, and corrects the changelog, integration guide and configuration guide sentences that repeated the stale claims. + +## Revision record + +| Date | Change | +| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-14 | First draft. | +| 2026-10-15 | Fixes from review round 1. The stale modules crate and deploy rules claims, the stages that refuse a mistake, the gap and open question for a `type` nothing offers, the breaking removal of `s3_sig_v4`, failure answers, rollout, the adapters that run each path, the missing tests and the review log. | From df5eab33573c30e02d8f66620c8d3dc02a630299 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 15 Oct 2026 17:54:06 +0100 Subject: [PATCH 6/7] The origin signing design takes its round 2 corrections Review round 2 found that the specs pull request is now IABTechLab/trusted-server#1342, because #1084 was closed, and three statements to tighten. The design now names #1342, says that the preflight signs without calling remove_namespace_headers, which makes no difference for s3_sigv4, says that GET /health on Fastly and Spin and the Fastly debug probes are still answered when an adapter cannot build its state, and limits the missing test for an empty session_token to deploy validation. The Tinybird entry in the changelog no longer names the deleted modules crate. --- CHANGELOG.md | 2 +- .../specs/2026-10-14-origin-signing-design.md | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d58fc5d2ff..28ac5e3988 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking for a build that links its own crates:** a source's `auth` table names an origin signer a crate offers. Core keeps the interface, `OriginSigner` in `trusted_server_core::origin_auth`, and signs the publisher's origin requests and each asset route's through it where it signed before. `type` names the signer, `origin_query` is read by core, and the rest of the table is the signer's. The `s3_sigv4` signer moves to `crates/shared/aws`, keeping its settings, defaults and secret-store key references, and every adapter and the `ts` tool link the crate's builder, so a stock build still offers it. A build that composes its own modules adds `trusted_server_shared_aws::builder()`, or a `type` of `s3_sigv4` is refused at startup naming the signers on offer. A table the signer cannot use is refused at startup and by `ts config validate` naming the setting. The deprecated spelling `s3_sig_v4` is no longer read. **For crates outside core**, `AssetOriginAuth` and `S3SigV4AuthConfig` are replaced by `OriginAuth`, `SourceLocation::auth` gives an `OriginSigning`, `OriginSettings::location` and `OriginSource::new` take the signers on offer and the setting's name, and a crate offers a signer with `IntegrationBuilder::with_origin_signer`. - **Breaking for crates outside core:** an adapter answers how it names a backend and bounds its transport timers through `PlatformBackend` alone, which gains `auction_backend_budget` and `transport_timeout_bucket_count` with defaults, and describes itself to auction plan validation through an `AuctionTargetDescriptor` it builds from its name, its backend implementation and its `AuctionTargetCapabilities`. `AuctionPlan::validate_for_target` takes that descriptor, and `BackendNamingPolicy`, `AuctionTargetId`, `PredictedBackend`, `BackendNamingError` and `PlatformBackend::naming_policy` are gone from core, because core names no hosting platform. Each adapter carries its own naming policy beside the code that registers its backends, and the names a deployment sees are unchanged. - The adapter declares the headers its platform reads a shared cache policy from. Core reads `Surrogate-Control` and `CDN-Cache-Control` for every platform, and the Fastly and Cloudflare adapters declare `fastly-surrogate-control` and `cloudflare-cdn-cache-control` as they build their state, so each is still stripped from every response core makes private. Responses carry the headers they carried. **For crates outside core**, `EdgeCacheHeader::CloudflareCdnCacheControl` is `EdgeCacheHeader::Platform(name)`, `EDGE_CACHE_HEADER_NAMES` is `cache_policy::edge_cache_header_names()`, the two platform header constants leave core, and an adapter declares its platform's headers with `platform::declare_platform_cache_headers`. -- Auction telemetry is an analytics module from a crate of its own. `[analytics]` is read as every module type's section is, and the Tinybird module, its settings and its checks live in `crates/analytics/tinybird`, offered by `crates/trusted-server-modules` to every adapter, where the Fastly adapter alone built it. The settings a deployment writes do not change. A table missing its host or token, a key the table does not know and a reserved setting turned on are refused when the deployment starts and by `ts config validate`, naming the table, where they were refused as the settings loaded. A `[tinybird]` table at the top level is refused as the section of a module type nothing supplies. **For crates outside core**, a registration declares the sink it supplies with `IntegrationRegistrationBuilder::with_analytics_module`, and `IntegrationRegistry::analytics_module` gives the one `[analytics] module` selected. `Settings::analytics_module`, `AnalyticsConfig` and `TinybirdSettings` leave core. +- Auction telemetry is an analytics module from a crate of its own. `[analytics]` is read as every module type's section is, and the Tinybird module, its settings and its checks live in `crates/analytics/tinybird`, linked by every adapter, where the Fastly adapter alone built it. The settings a deployment writes do not change. A table missing its host or token, a key the table does not know and a reserved setting turned on are refused when the deployment starts and by `ts config validate`, naming the table, where they were refused as the settings loaded. A `[tinybird]` table at the top level is refused as the section of a module type nothing supplies. **For crates outside core**, a registration declares the sink it supplies with `IntegrationRegistrationBuilder::with_analytics_module`, and `IntegrationRegistry::analytics_module` gives the one `[analytics] module` selected. `Settings::analytics_module`, `AnalyticsConfig` and `TinybirdSettings` leave core. - **Breaking for crates outside core:** core's auction names no protocol. A demand source implements `CompiledDemand`, which builds its own request from the routed input (`build_request`, answering the body, the headers, the slot ids and any state kept for the answer) and reads its own answer, and core sends what it built. OpenRTB 2.6 is the protocol module `crates/auction-protocol/openrtb`, where the request builder, the response reader, the wire types with their extension objects, and `OpenRtbDemand`, the hook trait a demand implementation that speaks OpenRTB fills in, now live, run through core's seam by `OpenRtbDriver`. `auction::openrtb` and core's `openrtb` module are gone, and core's own manifest keeps the generated types only for the response `POST /auction` returns. The dimension admission helpers are `auction::admission`, `apply_notification_policy` and `ignored_bidder_params_count` are `auction::provider`, and `Eid` and `Uid` are `ec::eids`. A deployment's settings do not change. - **Breaking:** The ad server this repository ships is `ad-server.iabtl`, from `crates/ad-server/iabtl` as the package `trusted-server-ad-server-iabtl`, where it was `ad-server.mock` from `crates/ad-server/mock`. A deployment that selected it writes `[ad-server] module = "iabtl"` and `[ad-server.iabtl]` where it wrote `module = "mock"` and `[ad-server.mock]`, and an `implementation = "ad-server.mock"` line names `ad-server.iabtl`. The seam probe, the test fixture that proves the integration seam from outside core, is `crates/testing/iabtl` as the package `trusted-server-testing-iabtl`, selected as `testing.iabtl`, where it was `crates/testing/seam-probe` selected as `testing.seam-probe`, with its shared table `[shared.iabtl]`, its loading cache `loading-cache.iabtl` and its declared cache `iabtl-answers` named the same way. The module that adds sellers to `ads.txt` is `ads-txt.iabtl`, from `crates/ads-txt/iabtl` as the package `trusted-server-ads-txt-iabtl`, where it was `ads-txt.sellers` from `crates/ads-txt/sellers`. A deployment that selected it writes `[ads-txt] modules = ["iabtl"]` and `[ads-txt.iabtl]` where it wrote `modules = ["sellers"]` and `[ads-txt.sellers]`, and the `[[fetch]]` entry on `/ads.txt` names the middleware `ads-txt.iabtl` where it named `ads-txt.sellers`. Each module describes the middleware it supplies on its builder with `IntegrationBuilder::with_middleware`, as `SuppliedMiddleware` values naming the middleware, the phase an entry names it in and the media type the entry covers, and `trusted_server_core::integrations::supplied_middleware(builders)` lists what a build supplies in the order the registry registers them, core's robots.txt and service version middleware included. The registry refuses a registration whose middleware contradicts its builder's description, in a phase it does not run in or for a media type it does not work on. **Breaking for crates outside core:** `crates/trusted-server-modules` is gone. Each adapter lists the module builders it links in its own `app.rs`, as `shipped_module_builders()`, with `module_builders_with(extra)` putting a deployment's builders after them, and the `ts` tool lists the builders it links and writes its entries and a module's section from what they describe. A deployment that took its list from `trusted_server_modules::builders()` or `builders_with(...)` names the crates it links instead, or takes an adapter's list. - **Breaking for crates outside core:** the consent decoders live in the crates named for their standards. The TCF string decoder is in `trusted-server-permission-signal-tcf`, the GPP string decoder in `trusted-server-permission-signal-gpp`, and the US Privacy string decoder in `trusted-server-permission-signal-us-privacy`, and each module reads its own string from the request through `PermissionSignalModule::read`, once, and reports a `SignalRecord`. `ConsentContext` is the records the configured modules reported and the request's jurisdiction, read by scheme name, with nothing of any standard in it, and a reader that needs a string in its standard's own form asks that standard's crate (`tc_string`, `applies`, `gpp_string`, `section_ids`, `us_privacy_string`, `signaled`). The server-side auction gate reads the resolved permissions, running the auction where `advertising_marketing.first_party.contextual` is set. `[consent]` keeps its keys, with `mode` read by core and the rest read by the modules, and `signals..authoritative` in `permissions.yaml` is read for any scheme. The identity graph record writes every vouched signal under its scheme, in the shape it wrote before for the two it stored. `CONSENT_COOKIE_NAMES`, `forward_cookie_header`, the consent cookie constants, `ConsentForwardingMode` (now the Prebid Server crate's), the `GdprConsent` error and the never-filled Additional Consent field and `additional_consent` policy flag are removed. diff --git a/docs/superpowers/specs/2026-10-14-origin-signing-design.md b/docs/superpowers/specs/2026-10-14-origin-signing-design.md index 081fc573f5..a312cafb7f 100644 --- a/docs/superpowers/specs/2026-10-14-origin-signing-design.md +++ b/docs/superpowers/specs/2026-10-14-origin-signing-design.md @@ -2,7 +2,7 @@ **Status:** Proposed, 2026-10-14. Raised as draft pull request jwrosewell/trusted-server#43, and as draft pull request IABTechLab/trusted-server#1391, for Tech Lab review. **Author:** 51Degrees (contributed), for Tech Lab review -**Related specs:** `2026-10-14-source-modules-design.md`, `2026-10-13-publisher-object-storage-origin-design.md`, `2026-05-19-asset-s3-auth-fastly-io-design.md` and `2026-10-14-no-modules-crate-design.md`, all in IABTechLab/trusted-server#1084, which holds every design document of the stack +**Related specs:** `2026-10-14-source-modules-design.md`, `2026-10-13-publisher-object-storage-origin-design.md`, `2026-05-19-asset-s3-auth-fastly-io-design.md` and `2026-10-14-no-modules-crate-design.md`, all in IABTechLab/trusted-server#1342, which holds every design document of the stack **Related PRs:** jwrosewell/trusted-server#43, IABTechLab/trusted-server#1391 **Issues:** IABTechLab/trusted-server#1338 **Last updated:** 2026-10-15 @@ -19,7 +19,7 @@ So core knows one vendor's scheme. A signed read from another vendor's storage, **Core keeps an origin authentication interface.** `trusted_server_core::origin_auth` declares `OriginSigner`, which signs the headers of a request for a method on a final address, and removes every header in the scheme's own namespace so a reader's header there cannot reach the origin beside the signature. Core calls it where it signed before, in the asset route path, in the preflight that path runs ahead of the image optimizer, and in the publisher path, through the `OriginSigning` a `SourceLocation` carries. The publisher path is core's own, so it signs on all four adapters. Asset routes and the preflight run on the Fastly adapter alone, because only that adapter calls `handle_asset_proxy_request`. -Three things differ from what core did before. First, the `HEAD` preflight, which asked an S3 origin for the object before the image optimizer did, now runs for any signed asset route that uses the optimizer, whatever its signer. Second, removing the headers in the signer's namespace is each signer's duty, so core calls `remove_namespace_headers` before `sign` on the publisher path and the asset route path, and does not check afterwards that none is left. Third, the asset route path's call to `remove_namespace_headers` is new, because that call was made on the publisher path alone. The asset route path starts its outbound headers from eleven named request headers (`ASSET_PROXY_FORWARD_HEADERS` in `crates/trusted-server-core/src/proxy.rs`) and none of them is in the `x-amz-` namespace, so for `s3_sigv4` the call removes nothing. +Three things differ from what core did before. First, the `HEAD` preflight, which asked an S3 origin for the object before the image optimizer did, now runs for any signed asset route that uses the optimizer, whatever its signer. Second, removing the headers in the signer's namespace is each signer's duty, so core calls `remove_namespace_headers` before `sign` on the publisher path and the asset route path, and does not check afterwards that none is left. Third, the asset route path's call to `remove_namespace_headers` is new, because that call was made on the publisher path alone. The asset route path starts its outbound headers from eleven named request headers (`ASSET_PROXY_FORWARD_HEADERS` in `crates/trusted-server-core/src/proxy.rs`) and none of them is in the `x-amz-` namespace, so for `s3_sigv4` the call removes nothing. The preflight signs its `HEAD` and its `GET` without calling `remove_namespace_headers`, which also makes no difference for `s3_sigv4`, because it signs copies of those same headers and `Host`. **A source's `auth` table names the signer.** The table keeps its shape. `type` names the signer, `origin_query` says whether the reader's query is kept on the signed request, and every other key belongs to the signer named. @@ -54,7 +54,7 @@ What leaves core: the signer, its settings type, the credential fields in core's 2. The deploy check, which `ts config validate` and `ts config push` run on the settings that were read, looks at each table whose `type` is a signer that the builders linked into the `ts` tool offer. It refuses a secret setting that is empty, only spaces or not text, naming its path, and it builds the signer from the table, so whatever the signer cannot use is refused naming the setting. It leaves alone a table whose `type` none of those builders offers. 3. An adapter's start builds the adapter's state, and with it the signer for every table, so that stage refuses what the signer cannot use and also a `type` that no builder the adapter links offers, naming the signers on offer. -**The gap.** A `type` that could be a module name but that no builder offers passes the first two stages and is refused only by the third. A misspelled `type`, the removed spelling `s3_sig_v4`, and the name of a signer that the `ts` tool does not link all pass `ts config validate` and `ts config push`, are stored, and then stop the adapter when it builds its state. That adapter answers each request with its startup error response, which is a `500` with the body "An internal error occurred" on Fastly, Axum and Cloudflare and a generic `503` on Spin, and it logs the full error. Before this change core refused any `type` other than `s3_sigv4` and `s3_sig_v4` while the settings were read, so the gap is new. The skip is deliberate, and the test `deploy_validation_leaves_a_signer_s_settings_alone_without_its_builder` in `crates/trusted-server-core/src/config.rs` pins it for a table whose signer's builder is not among those offered. +**The gap.** A `type` that could be a module name but that no builder offers passes the first two stages and is refused only by the third. A misspelled `type`, the removed spelling `s3_sig_v4`, and the name of a signer that the `ts` tool does not link all pass `ts config validate` and `ts config push`, are stored, and then stop the adapter when it builds its state. That adapter answers each request with its startup error response, which is a `500` with the body "An internal error occurred" on Fastly, Axum and Cloudflare and a generic `503` on Spin, and it logs the full error. The exceptions are `GET /health`, which Fastly and Spin still answer with `200`, and the debug probes that Fastly answers before it builds the application. Before this change core refused any `type` other than `s3_sigv4` and `s3_sig_v4` while the settings were read, so the gap is new. The skip is deliberate, and the test `deploy_validation_leaves_a_signer_s_settings_alone_without_its_builder` in `crates/trusted-server-core/src/config.rs` pins it for a table whose signer's builder is not among those offered. **Open question for the reviewer.** Should deploy validation close this gap or keep it? Closing the gap means refusing a `type` that no builder the `ts` tool links offers, so a misspelled `type` and the removed `s3_sig_v4` fail `ts config validate`. A deployment whose adapter links a signer of its own could then no longer push with the stock `ts` tool, which does not link that crate. The integration guide already describes the same limit for a module a deployment adds. Keeping the gap leaves a signer of the deployment's own pushable with the stock `ts` tool, and leaves a mistaken `type` to stop the adapter when it builds its state. @@ -89,12 +89,13 @@ Four cases have no test that can fail: 1. The asset route test that sends a reader's header in the signer's namespace cannot fail, because the route builds its outbound headers from eleven named request headers and that header is never among them. The test passes with the route's call to `remove_namespace_headers` removed, and no test gives that call a header it would remove. 2. The test `deploy_validation_leaves_a_signer_s_settings_alone_without_its_builder` in `crates/trusted-server-core/src/config.rs` shows deploy validation leaving a table alone when no builder offers its signer, with the core fixture signer and a blank key name. No test sends deploy validation a `type` that names no signer at all, such as a misspelling or `s3_sig_v4`, or shows the same settings then refused when the adapter builds its state. -3. No test gives an `s3_sigv4` table an empty `session_token`. Its refusal is tested only through the core fixture signer's `token_name`, a setting declared the same way. +3. No deploy validation test gives an `s3_sigv4` table an empty `session_token`. The refusal is tested only through the core fixture signer's `token_name`, a setting declared the same way. 4. No test in the `ts` tool's crate names the AWS builder or an `s3_sigv4` table, so nothing fails if the builder leaves the list in `linked_builders()`, and `ts config validate` would then skip every `s3_sigv4` table. ## Review log 1. **Round 1, 2026-10-15.** A review in a fresh session, against `split/48-aws-signing` at `a3c17125b`. Issues found, two of them blocking. The section on AWS signing named a modules crate that no longer exists and deploy rules the builder does not carry, and the spec left out that a `type` nothing offers passes `ts config validate` and `ts config push` and only stops the adapter at startup, while it listed the removed `s3_sig_v4` spelling under what is not in the change. This revision corrects both, states which stage refuses each mistake, what a failed signature answers, the rollout, the adapters that run each path, the settings that change and the tests that are missing or cannot fail, leaves closing the gap as an open question, and corrects the changelog, integration guide and configuration guide sentences that repeated the stale claims. +2. **Round 2, 2026-10-15.** A lighter review in a fresh session, against this page at `5c43f6f9a`, of the round 1 fixes and the new text. No blocking issues once the pull request number was corrected. This revision names IABTechLab/trusted-server#1342 as the specs pull request, says that the preflight signs without calling `remove_namespace_headers`, leaves `GET /health` and the Fastly debug probes out of the startup error answer, limits the missing test for an empty `session_token` to deploy validation, and corrects the Tinybird entry in the changelog. ## Revision record @@ -102,3 +103,4 @@ Four cases have no test that can fail: | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 2026-10-14 | First draft. | | 2026-10-15 | Fixes from review round 1. The stale modules crate and deploy rules claims, the stages that refuse a mistake, the gap and open question for a `type` nothing offers, the breaking removal of `s3_sig_v4`, failure answers, rollout, the adapters that run each path, the missing tests and the review log. | +| 2026-10-15 | Corrections from review round 2. The specs pull request number, the preflight's signing without removing namespace headers, the requests an adapter answers whatever its startup error, the deploy validation test for an empty token and the Tinybird changelog entry. | From 3787507eca74c8c4adfadf38c7b66661df42661e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 28 Oct 2026 11:30:24 +0000 Subject: [PATCH 7/7] The asset cache's signed route test signs with the fixture signer --- crates/trusted-server-core/src/proxy.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 00de476a65..c2970f5dad 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -5500,7 +5500,7 @@ mod tests { stub.push_response(200, b"ok".to_vec()); let module = TestAssetCache::asking_the_origin(); let settings = create_test_settings(); - let mut route = test_s3_image_optimizer_route(); + let mut route = test_signed_image_optimizer_route(); route.image_optimizer = None; let mut request = build_http_request(Method::GET, "https://www.example.com/.images/a.css"); @@ -5510,7 +5510,7 @@ mod tests { ); let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(test_s3_secrets()), + HashMapSecretStore::new(test_signed_route_secrets()), Arc::clone(&stub) as Arc, ) .with_asset_cache_module(Arc::clone(&module) as _); @@ -5537,9 +5537,13 @@ mod tests { .and_then(|value| value.to_str().ok()) .unwrap_or_default() .to_owned(); - assert!( - sent.starts_with("AWS4-HMAC-SHA256"), - "the origin request carries the route's own credentials, not the reader's: {sent}" + assert_eq!( + sent, + signed_route_authorization( + &Method::GET, + "https://bucket.storage.example.com/.images/a.css" + ), + "the origin request carries the route's own credentials, not the reader's" ); let forwarded_headers = super::ASSET_PROXY_FORWARD_HEADERS; let forwarded: Vec<&str> = forwarded_headers @@ -5551,7 +5555,7 @@ mod tests { forwarded.contains(&name.as_str()) || name == header::HOST || name == header::AUTHORIZATION - || name.as_str().starts_with("x-amz-"), + || name.as_str().starts_with("x-fixture-"), "the origin request carries only what the route forwards and signs: {name}" ); }