From 99f490e7a4af2866d3f7ac11e1aa54054c3a01a4 Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 15:58:59 +0000 Subject: [PATCH 01/15] Add an optional platform timer to runtime services --- .../trusted-server-core/src/platform/mod.rs | 3 ++ .../src/platform/test_support.rs | 29 +++++++++++++++- .../trusted-server-core/src/platform/timer.rs | 20 +++++++++++ .../trusted-server-core/src/platform/types.rs | 34 ++++++++++++++++++- 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 crates/trusted-server-core/src/platform/timer.rs diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 50fd3ca6cf..2f40b03e22 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -15,6 +15,7 @@ //! - [`PlatformTemplateCache`] — shared transformed-template caching //! - [`PlatformValueCache`], a shared cache of small values with lifetimes //! - [`PlatformLoadingCache`], a cache that loads a missing value once per key +//! - [`PlatformTimer`], an optional timer for bounding asynchronous work //! //! ## Platform-Agnostic Components //! @@ -57,6 +58,7 @@ mod template_cache; reason = "test doubles, offered to other crates' tests under the test-utils feature" )] pub mod test_support; +mod timer; mod traits; mod types; mod value_cache; @@ -101,6 +103,7 @@ pub use template_cache::{ pub use traits::{ GeoModule, PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore, }; +pub use timer::PlatformTimer; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, StoreName, diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 20f865f0ed..711175873a 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -12,7 +12,7 @@ use super::{ ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformCacheIntent, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformImageOptimizerOptions, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, - PlatformSelectResult, RuntimeServices, StoreId, StoreName, + PlatformSelectResult, PlatformTimer, RuntimeServices, StoreId, StoreName, }; pub struct NoopConfigStore; @@ -61,6 +61,33 @@ impl PlatformSecretStore for NoopSecretStore { } } +/// A timer whose sleeps have always already elapsed, recording each requested +/// duration. Work still pending when its deadline is first raced misses it. +#[derive(Default)] +pub struct ElapsedTimer { + pub requested: Mutex>, +} + +#[async_trait::async_trait(?Send)] +impl PlatformTimer for ElapsedTimer { + async fn sleep(&self, duration: Duration) { + self.requested + .lock() + .expect("should lock requested durations") + .push(duration); + } +} + +/// A timer that never fires. +pub struct StoppedTimer; + +#[async_trait::async_trait(?Send)] +impl PlatformTimer for StoppedTimer { + async fn sleep(&self, _duration: Duration) { + futures::future::pending::<()>().await; + } +} + pub struct HashMapConfigStore { data: HashMap, } diff --git a/crates/trusted-server-core/src/platform/timer.rs b/crates/trusted-server-core/src/platform/timer.rs new file mode 100644 index 0000000000..e08603588d --- /dev/null +++ b/crates/trusted-server-core/src/platform/timer.rs @@ -0,0 +1,20 @@ +use std::time::Duration; + +/// A timer that can wake a pending future after a delay. +/// +/// Optional on [`crate::platform::RuntimeServices`]: its presence is the +/// capability. An adapter provides one only when its runtime can fire a timer +/// while other work of the request is pending. A runtime whose host calls +/// block the guest could not fire one in the middle of a call, so its adapter +/// provides none, and work there is bounded by the transport timeouts alone. +/// +/// Uses `?Send` futures, matching [`crate::platform::PlatformHttpClient`] and +/// [`crate::platform::PlatformKvStore`], because the WebAssembly timers are +/// `!Send`. +#[async_trait::async_trait(?Send)] +pub trait PlatformTimer: Send + Sync { + /// Completes once `duration` has elapsed. + /// + /// Dropping the returned future before it completes cancels the timer. + async fn sleep(&self, duration: Duration); +} diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 8c3b960907..25266a330b 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -7,7 +7,7 @@ use crate::auction::telemetry::{AuctionTelemetrySink, NoopAuctionTelemetrySink}; use super::{ GeoModule, PlatformBackend, PlatformConfigStore, PlatformHttpClient, PlatformKvStore, - PlatformSecretStore, + PlatformSecretStore, PlatformTimer, }; use crate::ec::device::DeviceModule; use crate::ec::identifiers::IdentifierModule; @@ -203,6 +203,9 @@ pub struct RuntimeServices { pub(crate) geo: Arc, /// Auction telemetry sink. pub(crate) auction_telemetry_sink: Arc, + /// Wall-clock timer, when the runtime can fire one while request work is + /// pending. `None` means the platform has no such timer. + pub(crate) timer: Option>, /// Per-request client metadata extracted at the entry point. pub(crate) client_info: ClientInfo, /// Host-computed client signals (TLS JA4, HTTP/2), when the host @@ -350,6 +353,13 @@ impl RuntimeServices { &*self.auction_telemetry_sink } + /// Returns the platform timer, or `None` when the runtime cannot fire a + /// timer while request work is pending. + #[must_use] + pub fn timer(&self) -> Option<&dyn PlatformTimer> { + self.timer.as_deref() + } + /// Returns per-request client metadata (IP address, TLS details). #[must_use] pub fn client_info(&self) -> &ClientInfo { @@ -599,6 +609,15 @@ impl RuntimeServices { ..self } } + + /// Returns a clone of this instance with `timer` as the platform timer. + #[must_use] + pub fn with_timer(self, timer: Arc) -> Self { + Self { + timer: Some(timer), + ..self + } + } } impl fmt::Debug for RuntimeServices { @@ -625,6 +644,7 @@ pub struct RuntimeServicesBuilder { http_client: Option>, geo: Option>, auction_telemetry_sink: Option>, + timer: Option>, client_info: Option, host_signals: Option>, resolved_ec_module: Option>, @@ -648,6 +668,7 @@ impl RuntimeServicesBuilder { http_client: None, geo: None, auction_telemetry_sink: None, + timer: None, client_info: None, host_signals: None, resolved_ec_module: None, @@ -741,6 +762,16 @@ impl RuntimeServicesBuilder { self } + /// Set the platform timer, or `None` when the runtime has none. + /// + /// Only adapters whose runtime can fire a timer while request work is + /// pending provide one. Unset means no timer. + #[must_use] + pub fn timer(mut self, timer: Option>) -> Self { + self.timer = timer; + self + } + /// Set the per-request client metadata. #[must_use] pub fn client_info(mut self, client_info: ClientInfo) -> Self { @@ -856,6 +887,7 @@ impl RuntimeServicesBuilder { auction_telemetry_sink: self .auction_telemetry_sink .unwrap_or_else(|| Arc::new(NoopAuctionTelemetrySink)), + timer: self.timer, client_info: self .client_info .expect("should set client_info before building RuntimeServices"), From 561f0df68f036955fc3ac8f5ef28e6855c952dd3 Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:08:50 +0000 Subject: [PATCH 02/15] Open the Fastly runtime KV store when it is linked --- .../trusted-server-adapter-fastly/src/app.rs | 44 +++++++++- .../src/streaming_processor.rs | 84 +++++++++++++++++++ fastly.toml | 6 ++ 3 files changed, 131 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 7955c6cae9..47f4c8edda 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -85,6 +85,7 @@ use std::sync::Arc; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; use edgezero_adapter_fastly::context::FastlyRequestContext; +use edgezero_adapter_fastly::key_value_store::FastlyKvStore; use edgezero_core::app::{App, Hooks, StoreMetadata, StoresMetadata}; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; @@ -158,6 +159,23 @@ use crate::platform::{ // AppState // --------------------------------------------------------------------------- +/// The logical id the deployment's KV store is linked to the service under, +/// as `edgezero.toml` declares it. +const DEFAULT_KV_STORE_ID: &str = "trusted_server_kv"; + +/// Opens the KV store linked under [`DEFAULT_KV_STORE_ID`], or +/// [`UnavailableKvStore`] when the service links none, so a module that +/// needs the store passes its work by rather than failing the request. +fn open_default_kv_store() -> Arc { + match FastlyKvStore::open(DEFAULT_KV_STORE_ID) { + Ok(store) => Arc::new(store), + Err(err) => { + log::debug!("The KV store `{DEFAULT_KV_STORE_ID}` is not linked: {err}"); + Arc::new(UnavailableKvStore) + } + } +} + /// The stores a service reads and the key of its app-config blob. /// /// [`Self::logical`] gives the logical stores and key. The entry point puts @@ -398,7 +416,7 @@ pub(crate) fn build_state_with_registrations( Arc::new(trusted_server_core::auction::NoopAuctionTelemetrySink) as Arc }); - let default_kv_store = Arc::new(UnavailableKvStore) as Arc; + let default_kv_store = open_default_kv_store(); let loading_caches = trusted_server_core::integrations::loading_caches_from(&settings, integrations)?; @@ -1762,8 +1780,8 @@ impl Hooks for TrustedServerApp { ids: &[DEFAULT_CONFIG_STORE_ID], }), kv: Some(StoreMetadata { - default: "trusted_server_kv", - ids: &["trusted_server_kv"], + default: DEFAULT_KV_STORE_ID, + ids: &[DEFAULT_KV_STORE_ID], }), secrets: Some(StoreMetadata { default: DEFAULT_SECRET_STORE_ID, @@ -1866,6 +1884,26 @@ mod tests { assert_eq!(stores.secret_store_name.as_ref(), "trusted_server_secrets"); } + #[test] + fn app_state_opens_the_linked_kv_store() { + let state = app_state_for_settings(test_settings()); + + block_on( + state + .default_kv_store + .put_bytes("default-kv-open-test", Bytes::from_static(b"value")), + ) + .expect("should write to the linked KV store"); + let value = block_on(state.default_kv_store.get_bytes("default-kv-open-test")) + .expect("should read from the linked KV store"); + + assert_eq!( + value, + Some(Bytes::from_static(b"value")), + "should read back what was written to the store linked as trusted_server_kv" + ); + } + fn app_state_for_settings(settings: Settings) -> Arc { build_state_from_settings(settings).expect("should build app state from settings") } diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index ec5ba1ac8f..4c751ebdcd 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -22,6 +22,7 @@ use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; use std::rc::Rc; +use std::time::Duration; use brotli::Decompressor; use brotli::enc::BrotliEncoderParams; @@ -29,8 +30,10 @@ use brotli::enc::writer::CompressorWriter; use error_stack::{Report, ResultExt as _}; use flate2::read::ZlibDecoder; use flate2::write::{GzEncoder, ZlibEncoder}; +use futures::future::{Either, LocalBoxFuture}; use crate::error::TrustedServerError; +use crate::platform::RuntimeServices; /// Trait for streaming content processors pub trait StreamProcessor { @@ -50,6 +53,87 @@ pub trait StreamProcessor { /// Reset the processor state (useful for reuse) fn reset(&mut self) {} + + /// Start an optional asynchronous finish of this processor's final output. + /// + /// `held` is exactly what the final [`process_chunk`](Self::process_chunk) + /// call (`is_last = true`) returned: the bytes the processor emits when it + /// does no asynchronous work. The driver owns `held` and calls this at most + /// once, right after that final call, with the request's + /// [`RuntimeServices`]. + /// + /// Returning `None` (the default) emits `held` unchanged; no future is + /// created. Returning a finish makes the driver await it and emit its + /// output in place of `held`. The output is infallible: a processor absorbs + /// its own failures and returns the bytes to emit, typically `held` itself. + /// On a runtime with a [`PlatformTimer`](crate::platform::PlatformTimer), + /// the driver drops the finish once the deadline given to + /// [`AsyncFinish::new`] elapses and emits `held` instead. + /// + /// Only drivers that can await call this. A synchronous driver, such as + /// [`StreamingPipeline`], emits `held`. + fn async_finish<'a>( + &'a mut self, + held: &'a [u8], + services: &'a RuntimeServices, + ) -> Option> { + let _ = (held, services); + None + } +} + +/// Asynchronous work a [`StreamProcessor`] completes after its final chunk. +/// +/// Created by [`StreamProcessor::async_finish`]. The future is `?Send`, like +/// the platform services it awaits. +pub struct AsyncFinish<'a> { + deadline: Duration, + output: LocalBoxFuture<'a, Vec>, +} + +impl<'a> AsyncFinish<'a> { + /// Wrap `output`, which resolves to the bytes to emit, with the time the + /// driver may wait for it. + /// + /// `deadline` is measured from when the driver starts awaiting the finish. + /// It is a backstop: a processor should bound its own calls well within it. + #[must_use] + pub fn new(deadline: Duration, output: impl Future> + 'a) -> Self { + Self { + deadline, + output: Box::pin(output), + } + } +} + +/// Run `processor`'s optional asynchronous finish over `held`, its final output. +/// +/// Returns the finish output, or `held` when the processor has no finish. When +/// `services` has a timer, the finish races its deadline; on expiry the finish +/// future is dropped and `held` is returned. Without a timer the finish is +/// awaited to completion. +pub(crate) async fn finish_held( + processor: &mut dyn StreamProcessor, + held: Vec, + services: &RuntimeServices, +) -> Vec { + let deadline = { + let Some(AsyncFinish { deadline, output }) = processor.async_finish(&held, services) else { + return held; + }; + let Some(timer) = services.timer() else { + return output.await; + }; + match futures::future::select(output, timer.sleep(deadline)).await { + Either::Left((output, _)) => return output, + Either::Right(((), _)) => deadline, + } + }; + log::warn!( + "stream processor finish exceeded its {} ms deadline; emitting its held output", + deadline.as_millis() + ); + held } /// Compression type for the stream diff --git a/fastly.toml b/fastly.toml index 020b15ea28..bc8f06a7df 100644 --- a/fastly.toml +++ b/fastly.toml @@ -23,6 +23,12 @@ build = """ [local_server.backends] [local_server.kv_stores] + # The runtime KV store, linked as the logical `trusted_server_kv` that + # edgezero.toml declares. + [[local_server.kv_stores.trusted_server_kv]] + key = "placeholder" + data = "placeholder" + [[local_server.kv_stores.counter_store]] key = "placeholder" data = "placeholder" From 589838405c01e81e135bf6bd8dfb8fe0506bf703 Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:08:50 +0000 Subject: [PATCH 03/15] End middleware stream processors with an awaited asynchronous finish A stream processor may implement StreamProcessor::async_finish over the bytes its last process_chunk call returned. StreamProcessor::finish ends a processor once; a processor made of others ends each in order, so each is handed what the one before it finished with. Every publisher path that can await now ends its processor this way: the no-hold pipeline, both auction-hold loops, the serve pass on a shared template and the lazy stream. A body read whole takes the same chunk pipeline, and the synchronous stream_publisher_body path is removed. A middleware is also told the response status and whether the origin marked the response as one reader's, read before core writes its own cache policy. --- crates/middleware/51degrees/src/lib.rs | 2 + .../benches/html_processor_bench.rs | 2 + .../trusted-server-core/src/html_processor.rs | 66 +++ .../src/integrations/js_asset_proxy.rs | 2 + crates/trusted-server-core/src/middleware.rs | 12 + .../src/middleware/tests.rs | 2 + crates/trusted-server-core/src/publisher.rs | 523 +++++++++--------- .../src/streaming_processor.rs | 44 +- 8 files changed, 381 insertions(+), 272 deletions(-) diff --git a/crates/middleware/51degrees/src/lib.rs b/crates/middleware/51degrees/src/lib.rs index 4b5a557ef8..bd5bf849f8 100644 --- a/crates/middleware/51degrees/src/lib.rs +++ b/crates/middleware/51degrees/src/lib.rs @@ -658,6 +658,8 @@ mod tests { request_host: "www.example.com", request_scheme: "https", origin_host: "origin.example.com", + status: http::StatusCode::OK, + origin_private: false, platform: test_support::platform(), document_state: state, max_buffered_script_bytes: 1 << 20, diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index e650dc0782..962001fc22 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -18,6 +18,8 @@ fn make_config() -> HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, prepared: std::sync::Arc::new(trusted_server_core::middleware::Prepared::default()), + status: http::StatusCode::OK, + origin_private: false, request_state: trusted_server_core::integrations::IntegrationRequestState::default(), // The benchmark measures URL rewriting, not ad injection, and // `ad_slots_script` is `None` here — matching the previous behaviour, diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 723e55a234..44929b7825 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -7,6 +7,8 @@ use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use futures::future::LocalBoxFuture; +use http::StatusCode; use lol_html::{ EndTagHandler, Settings as RewriterSettings, element, end, html_content::{ContentType, EndTag, TextChunk, TextType}, @@ -22,6 +24,7 @@ use crate::middleware::{ ElementHandler, MatchedAttribute, MiddlewareChain, MiddlewareContext, MiddlewarePlan, Prepared, ResponseHeader, TextHandler, }; +use crate::platform::RuntimeServices; use crate::prefix::{PUBLIC_PREFIX_ATTRIBUTE, PublicPrefix}; use crate::publisher::build_empty_bids_script; use crate::settings::Settings; @@ -32,6 +35,8 @@ mod script_text; use script_text::ScriptTextGuard; +/// The HTML rewriter followed by the middleware's stream processors, each +/// handed what the one before it produced. struct HtmlWithStreamingProcessors { inner: Box, processors: Vec>, @@ -52,6 +57,24 @@ impl StreamProcessor for HtmlWithStreamingProcessors { processor.reset(); } } + + /// Ends the rewriter once and then each processor in order, so a + /// processor is handed its last chunk only after the one before it has + /// finished, its asynchronous finish included. + fn finish<'a>( + &'a mut self, + chunk: &'a [u8], + services: &'a RuntimeServices, + ) -> LocalBoxFuture<'a, Result, io::Error>> { + Box::pin(async move { + let mut output = self.inner.finish(chunk, services).await?; + for processor in &mut self.processors { + let finished = processor.finish(&output, services).await?; + output = finished; + } + Ok(output) + }) + } } /// What the `` seam injects. @@ -125,6 +148,10 @@ pub struct HtmlProcessorConfig { /// What the middleware prepared for this document, see /// [`Middleware::prepare`](crate::middleware::Middleware::prepare). pub prepared: Arc, + /// See [`MiddlewareContext::status`]. + pub status: StatusCode, + /// See [`MiddlewareContext::origin_private`]. + pub origin_private: bool, } impl HtmlProcessorConfig { @@ -151,6 +178,8 @@ impl HtmlProcessorConfig { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, } } @@ -224,6 +253,15 @@ impl HtmlProcessorConfig { self.prepared = prepared; self } + + /// Sets the status of the response the document is the body of, and + /// whether the origin marked that response as one reader's. + #[must_use] + pub fn with_response(mut self, status: StatusCode, origin_private: bool) -> Self { + self.status = status; + self.origin_private = origin_private; + self + } } /// The chunk of text being rewritten, as the last handler to change it left @@ -301,6 +339,8 @@ pub fn create_html_processor_with_middleware( request_host: &config.request_host, request_scheme: &config.request_scheme, origin_host: &config.origin_host, + status: config.status, + origin_private: config.origin_private, platform: config.integrations.platform(), document_state: &document_state, max_buffered_script_bytes: config.max_buffered_body_bytes, @@ -313,6 +353,8 @@ pub fn create_html_processor_with_middleware( request_host: &config.request_host, request_scheme: &config.request_scheme, origin_host: &config.origin_host, + status: config.status, + origin_private: config.origin_private, platform: config.integrations.platform(), request_state: &config.request_state, max_buffered_script_bytes: config.max_buffered_body_bytes, @@ -336,6 +378,10 @@ pub struct ReaderDocument<'a> { /// The host the document names as its own, being the `Host` its source /// sends. pub origin_host: &'a str, + /// See [`MiddlewareContext::status`]. + pub status: StatusCode, + /// See [`MiddlewareContext::origin_private`]. + pub origin_private: bool, /// What the adapter reports about the platform the deployment runs on. pub platform: &'a PlatformIdentity, /// What modules left on this reader's request for their page changes. @@ -372,6 +418,8 @@ pub fn create_serve_processor( request_host: document.request_host, request_scheme: document.request_scheme, origin_host: document.origin_host, + status: document.status, + origin_private: document.origin_private, platform: document.platform, document_state: &document_state, max_buffered_script_bytes: document.max_buffered_script_bytes, @@ -1240,6 +1288,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_owned(), request_host: "test.example.com".to_owned(), @@ -1991,6 +2041,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -2071,6 +2123,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -2112,6 +2166,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -2154,6 +2210,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.test-publisher.example.com".to_string(), request_host: request_host.to_string(), @@ -2210,6 +2268,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -2244,6 +2304,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -2280,6 +2342,8 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, request_state: IntegrationRequestState::default(), } } @@ -2490,6 +2554,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::Marker(MARKER.to_string()), origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs index 69d9fca9ab..76e9103360 100644 --- a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -687,6 +687,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: std::sync::Arc::new(crate::middleware::Prepared::default()), + status: http::StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "publisher.example.com".to_string(), diff --git a/crates/trusted-server-core/src/middleware.rs b/crates/trusted-server-core/src/middleware.rs index bf7717dc6a..f8555ff0fe 100644 --- a/crates/trusted-server-core/src/middleware.rs +++ b/crates/trusted-server-core/src/middleware.rs @@ -144,6 +144,16 @@ pub struct MiddlewareContext<'a> { /// sends, the same for a page served from the store as for one fetched /// for this request. pub origin_host: &'a str, + /// The status of the response the document is the body of. A page from + /// the store of shared templates was stored from a `200`. + pub status: StatusCode, + /// Whether the origin marked its response as one reader's, by setting a + /// cookie or by a `Cache-Control` that forbids shared storage. Read from + /// the origin's own headers, before core writes its cache policy on the + /// response, so a middleware that sends the document elsewhere can leave + /// a reader's own page alone. A page from the store of shared templates + /// was never one reader's. + pub origin_private: bool, /// What the adapter reports about the platform the deployment runs on, /// as the attestation evidence carries it. pub platform: &'a PlatformIdentity, @@ -1103,6 +1113,8 @@ pub mod test_support { request_host: REQUEST_HOST, request_scheme: "https", origin_host: ORIGIN_HOST, + status: http::StatusCode::OK, + origin_private: false, platform, document_state, max_buffered_script_bytes: 16 * 1024 * 1024, diff --git a/crates/trusted-server-core/src/middleware/tests.rs b/crates/trusted-server-core/src/middleware/tests.rs index 72814b6d26..fc493f74e1 100644 --- a/crates/trusted-server-core/src/middleware/tests.rs +++ b/crates/trusted-server-core/src/middleware/tests.rs @@ -67,6 +67,8 @@ fn reader(request_state: &IntegrationRequestState) -> ReaderDocument<'_> { request_host: REQUEST_HOST, request_scheme: "https", origin_host: ORIGIN_HOST, + status: StatusCode::OK, + origin_private: false, platform: test_support::platform(), request_state, max_buffered_script_bytes: 16 * 1024 * 1024, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8ebe52a4a6..94d633b9e4 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -11,12 +11,12 @@ //! must gate them out before the body enters the rewrite pipeline. //! //! **Note on platform coupling:** The handler boundaries use portable HTTP -//! types: [`handle_publisher_request`] and [`stream_publisher_body`] take and -//! return `http::Request`/`http::Response` over `EdgeBody`, and platform I/O is -//! reached through `RuntimeServices` rather than a platform SDK directly. The -//! streaming processor itself is generic: `process_response_streaming` writes -//! into any [`Write`] (a `Vec` for buffered routes, a streaming writer for -//! the streaming route). It is not a content-rewriting concern. +//! types: [`handle_publisher_request`] and [`stream_publisher_body_async`] +//! take and return `http::Request`/`http::Response` over `EdgeBody`, and +//! platform I/O is reached through `RuntimeServices` rather than a platform +//! SDK directly. The body pipeline itself is generic: it writes into any +//! [`Write`] (a `Vec` for buffered routes) or yields a lazy stream for the +//! streaming route. It is not a content-rewriting concern. use std::borrow::Cow; use std::io::Write; @@ -32,6 +32,7 @@ use error_stack::{Report, ResultExt}; use flate2::read::ZlibDecoder; use flate2::write::{GzEncoder, ZlibEncoder}; use futures::StreamExt as _; +use futures::future::LocalBoxFuture; use http::{HeaderValue, Method, Request, Response, StatusCode, Uri, header}; use sha2::Digest as _; use web_time::Instant; @@ -96,8 +97,8 @@ use crate::settings::{ use crate::source::{SourceLocation, SourceRequest}; use crate::source_cache::{Found, Lookup, SourceAddress, SourceCache, SourceVary}; use crate::streaming_processor::{ - BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, PipelineConfig, - STREAM_CHUNK_SIZE, StreamProcessor, StreamingPipeline, + BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, STREAM_CHUNK_SIZE, + StreamProcessor, }; use crate::streaming_replacer::create_url_replacer; @@ -261,19 +262,43 @@ impl BodyChunkSource { } } -fn process_and_encode_chunk( +fn process_and_encode_chunk( processor: &mut P, encoder: &mut BodyStreamEncoder, chunk: &[u8], - is_last: bool, process_error: &str, ) -> Result, Report> { let processed = processor - .process_chunk(chunk, is_last) + .process_chunk(chunk, false) .change_context(TrustedServerError::Proxy { message: process_error.to_string(), })?; + encode_processed(encoder, processed) +} + +/// Ends `processor`, awaiting any asynchronous finish of a middleware's +/// processor, and encodes what it emits last. +async fn finish_and_encode( + processor: &mut P, + encoder: &mut BodyStreamEncoder, + services: &RuntimeServices, +) -> Result, Report> { + let processed = + processor + .finish(&[], services) + .await + .change_context(TrustedServerError::Proxy { + message: "Failed to finalize processor".to_string(), + })?; + encode_processed(encoder, processed) +} + +/// Encodes processed bytes, or `None` when there is nothing to emit. +fn encode_processed( + encoder: &mut BodyStreamEncoder, + processed: Vec, +) -> Result, Report> { if processed.is_empty() { return Ok(None); } @@ -635,36 +660,6 @@ fn parse_single_module_filename( registry.js_module_id(stem) } -/// Parameters for processing response streaming. -struct ProcessResponseParams<'a> { - content_encoding: &'a str, - origin_host: &'a str, - origin_url: &'a str, - request_host: &'a str, - request_scheme: &'a str, - /// See [`OwnedProcessResponseParams::request_path`]. - request_path: &'a str, - settings: &'a Settings, - content_type: &'a str, - /// See [`OwnedProcessResponseParams::status`]. - status: StatusCode, - integration_registry: &'a IntegrationRegistry, - /// Head script carrying this request's permission state, or [`None`] under a - /// shared-template mode. See [`template_permissions_script`]. - permissions_script: Option<&'a str>, - ad_slots_script: Option<&'a str>, - ad_bids_state: &'a Arc>>, - /// See [`HtmlStreamProcessorParams::request_state`]. - request_state: &'a IntegrationRequestState, - /// See [`HtmlStreamProcessorParams::shared_template_authorized`]. - shared_template_authorized: bool, - /// See [`HtmlStreamProcessorParams::csp_nonce_observed`]. - csp_nonce_observed: Option<&'a Arc>, - prepared: &'a Arc, - /// See [`OwnedProcessResponseParams::public_prefix`]. - public_prefix: &'a crate::prefix::PublicPrefix, -} - struct PublisherBodyProcessor { inner: Box, /// What the serve middleware write on the response, taken by the @@ -679,6 +674,8 @@ struct TextResponse<'a> { /// The response's `Content-Type`, parameters and all. content_type: &'a str, status: StatusCode, + /// See [`MiddlewareContext::origin_private`](crate::middleware::MiddlewareContext::origin_private). + origin_private: bool, request_path: &'a str, request_host: &'a str, request_scheme: &'a str, @@ -744,6 +741,8 @@ fn middleware_stream_processors( request_host: response.request_host, request_scheme: response.request_scheme, origin_host: response.origin_host, + status: response.status, + origin_private: response.origin_private, platform: integration_registry.platform(), document_state: &document_state, max_buffered_script_bytes: response.max_buffered_script_bytes, @@ -898,6 +897,8 @@ async fn prepare_document( request_host: document.request_host, request_scheme: document.request_scheme, origin_host: document.origin_host, + status: document.status, + origin_private: document.origin_private, platform: integration_registry.platform(), document_state: &document_state, max_buffered_script_bytes: document.max_buffered_script_bytes, @@ -1004,6 +1005,27 @@ impl StreamProcessor for WithStreamProcessors { processor.reset(); } } + + /// Ends core's move of the address and then each processor in order, so + /// a processor is handed its last chunk only after the one before it has + /// finished, its asynchronous finish included. + fn finish<'a>( + &'a mut self, + chunk: &'a [u8], + services: &'a RuntimeServices, + ) -> LocalBoxFuture<'a, Result, std::io::Error>> { + Box::pin(async move { + let mut output = match &mut self.inner { + Some(inner) => inner.finish(chunk, services).await?, + None => chunk.to_vec(), + }; + for processor in &mut self.processors { + let finished = processor.finish(&output, services).await?; + output = finished; + } + Ok(output) + }) + } } impl PublisherBodyProcessor { @@ -1029,6 +1051,8 @@ impl PublisherBodyProcessor { { let (processor, headers) = create_html_stream_processor(HtmlStreamProcessorParams { origin_host: ¶ms.origin_host, + status: params.status, + origin_private: params.origin_private, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, request_path: ¶ms.request_path, @@ -1052,6 +1076,7 @@ impl PublisherBodyProcessor { &TextResponse { content_type: ¶ms.content_type, status: params.status, + origin_private: params.origin_private, request_path: ¶ms.request_path, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, @@ -1093,105 +1118,31 @@ impl StreamProcessor for PublisherBodyProcessor { fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, std::io::Error> { self.inner.process_chunk(chunk, is_last) } + + fn finish<'a>( + &'a mut self, + chunk: &'a [u8], + services: &'a RuntimeServices, + ) -> LocalBoxFuture<'a, Result, std::io::Error>> { + self.inner.finish(chunk, services) + } } -/// Process response body through the streaming pipeline. -/// -/// Selects the processor by content type, the HTML rewriter or the text -/// pass of the entries' middleware, and pipes chunks from `body` through it -/// into `output`. The caller decides what `output` is, a `Vec` for -/// buffered responses or a `StreamingBody` for streaming. +/// Pipes `body` through the processor its content type selects, the HTML +/// rewriter or the text pass of the entries' middleware, into `output`, and +/// ends the processor with its asynchronous finish. A body read whole and a +/// streamed one take the same chunk pipeline. /// /// # Errors /// /// Returns an error if processor creation or chunk processing fails. -fn process_response_streaming( - body: EdgeBody, - output: &mut W, - params: &ProcessResponseParams, - output_compression: Compression, -) -> Result, Report> { - let is_html = is_html_content_type(params.content_type); - log::debug!( - "process_response_streaming: content_type={}, content_encoding={}, is_html={}", - params.content_type, - params.content_encoding, - is_html - ); - - let compression = Compression::from_content_encoding(params.content_encoding); - let config = PipelineConfig { - input_compression: compression, - output_compression, - chunk_size: 8192, - }; - // Bound how much decoded gzip output may sit in the heap at once, using the - // same ceiling the buffered writer enforces on the rewritten output: a gzip - // bomb is then rejected mid-decode instead of materializing its full - // expansion first. The bound is per-step, so a large honest body still - // streams through and only the buffered writer judges the total — otherwise - // gzip would reject bodies the identity, deflate and brotli paths accept. - let max_pending_decoded_bytes = params.settings.publisher.max_buffered_body_bytes; - - let response_headers = if is_html { - let (processor, headers) = create_html_stream_processor(HtmlStreamProcessorParams { - origin_host: params.origin_host, - request_host: params.request_host, - request_scheme: params.request_scheme, - request_path: params.request_path, - settings: params.settings, - integration_registry: params.integration_registry, - permissions_script: params.permissions_script.map(str::to_string), - ad_slots_script: params.ad_slots_script.map(str::to_string), - ad_bids_state: params.ad_bids_state.clone(), - request_state: params.request_state.clone(), - shared_template_authorized: params.shared_template_authorized, - csp_nonce_observed: params.csp_nonce_observed.cloned(), - prepared: Arc::clone(params.prepared), - public_prefix: params.public_prefix, - deferred_inline_marker: None, - })?; - StreamingPipeline::new(config, processor) - .with_max_pending_decoded_bytes(max_pending_decoded_bytes) - .process(body_as_reader(body)?, output)?; - headers - } else { - let (processor, headers) = middleware_stream_processors( - &TextResponse { - content_type: params.content_type, - status: params.status, - request_path: params.request_path, - request_host: params.request_host, - request_scheme: params.request_scheme, - origin_host: params.origin_host, - request_state: params.request_state, - max_buffered_script_bytes: params.settings.publisher.max_buffered_body_bytes, - prepared: params.prepared, - }, - params.settings, - params.integration_registry, - )? - .into_processor( - params.origin_host, - params.origin_url, - params.request_host, - params.request_scheme, - ); - StreamingPipeline::new(config, processor) - .with_max_pending_decoded_bytes(max_pending_decoded_bytes) - .process(body_as_reader(body)?, output)?; - headers - }; - - Ok(response_headers) -} - async fn process_response_streaming_async( body: EdgeBody, output: &mut W, params: &OwnedProcessResponseParams, settings: &Settings, integration_registry: &IntegrationRegistry, + services: &RuntimeServices, ) -> Result, Report> { log::debug!( "process_response_streaming_async: content_type={}, content_encoding={}", @@ -1228,6 +1179,7 @@ async fn process_response_streaming_async( input_compression, output_compression, settings.publisher.max_buffered_body_bytes, + services, ) .await?; Ok(response_headers) @@ -1239,7 +1191,7 @@ async fn process_response_streaming_async( /// [`passthrough_finish_segments`]. Shared by the write-sink driver /// ([`process_body_chunks_async`]) and the lazy publisher body stream so the /// two no-hold paths cannot drift apart. -async fn passthrough_step( +async fn passthrough_step( source: &mut BodyChunkSource, decoder: &mut BodyStreamDecoder, encoder: &mut BodyStreamEncoder, @@ -1253,25 +1205,22 @@ async fn passthrough_step( return Ok(Some(Vec::new())); } let mut segments = Vec::new(); - if let Some(encoded) = process_and_encode_chunk( - processor, - encoder, - &decoded, - false, - "Failed to process chunk", - )? { + if let Some(encoded) = + process_and_encode_chunk(processor, encoder, &decoded, "Failed to process chunk")? + { segments.push(encoded); } Ok(Some(segments)) } -async fn process_body_chunks_async( +async fn process_body_chunks_async( body: EdgeBody, writer: &mut W, processor: &mut P, input_compression: Compression, output_compression: Compression, max_body_bytes: usize, + services: &RuntimeServices, ) -> Result<(), Report> { let mut decoder = BodyStreamDecoder::new(input_compression, max_body_bytes); let mut encoder = BodyStreamEncoder::new(output_compression); @@ -1285,7 +1234,9 @@ async fn process_body_chunks_async( } } - for encoded in passthrough_finish_segments(processor, &mut decoder, &mut encoder)? { + for encoded in + passthrough_finish_segments(processor, &mut decoder, &mut encoder, services).await? + { write_encoded_segment(writer, &encoded)?; } writer.flush().change_context(TrustedServerError::Proxy { @@ -1308,12 +1259,13 @@ fn write_encoded_segment( } /// Finalize a no-hold chunk pipeline: drain the decoder tail through the -/// processor, signal end-of-stream to the processor, and emit the encoder -/// trailer. Returns the encoded segments for the caller to emit. -fn passthrough_finish_segments( +/// processor, end the processor with its asynchronous finish, and emit the +/// encoder trailer. Returns the encoded segments for the caller to emit. +async fn passthrough_finish_segments( processor: &mut P, decoder: &mut BodyStreamDecoder, encoder: &mut BodyStreamEncoder, + services: &RuntimeServices, ) -> Result, Report> { let mut segments = Vec::new(); let decoded_tail = decoder.finish()?; @@ -1322,19 +1274,12 @@ fn passthrough_finish_segments( processor, encoder, &decoded_tail, - false, "Failed to process decoded tail", )? { segments.push(encoded); } - if let Some(encoded) = process_and_encode_chunk( - processor, - encoder, - &[], - true, - "Failed to finalize processor", - )? { + if let Some(encoded) = finish_and_encode(processor, encoder, services).await? { segments.push(encoded); } let trailer = encoder.finish()?; @@ -1585,8 +1530,9 @@ async fn hold_step_next_chunk( .map(Some) } -/// Drain the decoder tail at end of the origin stream, returning the prefix the -/// caller must emit before [`hold_finish_tail_segments`]. +/// Drain the decoder tail at end of the origin stream and end the processor, +/// returning the prefix the caller must emit before +/// [`hold_finish_tail_segments`]. /// /// A codec can hold document bytes back until its own finalization — the gzip /// decoder releases the remainder of the final member at `finish()` — and that @@ -1594,8 +1540,12 @@ async fn hold_step_next_chunk( /// collection keeps the invariant the mid-stream path already has: only the /// closing `` tail waits for the auction, never renderable content. /// -/// On decoder failure the pending auction is abandoned before the error is -/// returned. +/// The processor's end awaits each middleware processor's asynchronous finish +/// in order, so a seam token a processor held reaches the seam scanner here, +/// once the whole chain has finished. +/// +/// On decoder or processor failure the pending auction is abandoned before +/// the error is returned. async fn hold_finish_ready_segments( processor: &mut P, decoder: &mut BodyStreamDecoder, @@ -1613,18 +1563,18 @@ async fn hold_finish_ready_segments( let mut step = hold_step_decoded_chunk(processor, encoder, &decoded_tail, state, collect_refs).await?; - let final_processed = - match processor - .process_chunk(&[], true) - .change_context(TrustedServerError::Proxy { - message: "Failed to finalize processor".to_string(), - }) { - Ok(processed) => processed, - Err(err) => { - abandon_hold_auction(state, collect_refs.services, "stream_process_error").await; - return Err(err); - } - }; + let final_processed = match processor + .finish(&[], collect_refs.services) + .await + .change_context(TrustedServerError::Proxy { + message: "Failed to finalize processor".to_string(), + }) { + Ok(processed) => processed, + Err(err) => { + abandon_hold_auction(state, collect_refs.services, "stream_process_error").await; + return Err(err); + } + }; let final_ready = match state.hold.as_mut() { Some(seam) => seam.push(&final_processed), None => final_processed, @@ -1703,6 +1653,10 @@ async fn hold_finish_tail_segments( /// registry alive for as long as the processor. struct HtmlStreamProcessorParams<'a> { origin_host: &'a str, + /// See [`OwnedProcessResponseParams::status`]. + status: StatusCode, + /// See [`OwnedProcessResponseParams::origin_private`]. + origin_private: bool, request_host: &'a str, request_scheme: &'a str, /// See [`OwnedProcessResponseParams::request_path`]. @@ -1920,7 +1874,8 @@ fn create_html_stream_processor( .with_body_close(body_close) .with_csp_nonce_observer(csp_nonce_observed) .with_public_prefix(params.public_prefix.clone()) - .with_prepared(params.prepared); + .with_prepared(params.prepared) + .with_response(params.status, params.origin_private); // The middleware of the first [[fetch]] entry covering this path. let fetch = params.integration_registry.middleware_chain( @@ -1975,6 +1930,8 @@ fn reader_serve_processor( request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, origin_host, + status: params.status, + origin_private: params.origin_private, platform: integration_registry.platform(), request_state: ¶ms.request_state, max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, @@ -1985,14 +1942,15 @@ fn reader_serve_processor( } /// Runs the `[[serve]]` entry covering this request over a reader's whole -/// copy, assembled and not yet encoded, and writes its headers on the -/// response. -fn run_serve_pass( +/// copy, assembled and not yet encoded, ending it with its asynchronous +/// finish, and writes its headers on the response. +async fn run_serve_pass( bytes: Vec, params: &OwnedProcessResponseParams, settings: &Settings, integration_registry: &IntegrationRegistry, response: &mut Response, + services: &RuntimeServices, ) -> Result, Report> { let pass = reader_serve_processor(params, settings, integration_registry)?; write_response_headers(response, pass.response_headers); @@ -2000,7 +1958,8 @@ fn run_serve_pass( return Ok(bytes); }; processor - .process_chunk(&bytes, true) + .finish(&bytes, services) + .await .change_context(TrustedServerError::Proxy { message: "a serve middleware could not rewrite the page".to_string(), }) @@ -2011,10 +1970,22 @@ fn run_serve_pass( fn serve_chunk( serve: &mut Option>, bytes: Vec, - is_last: bool, ) -> Result, std::io::Error> { match serve { - Some(processor) => processor.process_chunk(&bytes, is_last), + Some(processor) => processor.process_chunk(&bytes, false), + None => Ok(bytes), + } +} + +/// The last piece of a streamed reader's copy through the `[[serve]]` entry, +/// when one covers the request, ending it with its asynchronous finish. +async fn serve_last_chunk( + serve: &mut Option>, + bytes: Vec, + services: &RuntimeServices, +) -> Result, std::io::Error> { + match serve { + Some(processor) => processor.finish(&bytes, services).await, None => Ok(bytes), } } @@ -2045,7 +2016,7 @@ pub enum PublisherResponse { response: Response, /// Origin body to be piped through the streaming pipeline. body: EdgeBody, - /// Parameters for [`process_response_streaming`]. + /// Parameters for [`stream_publisher_body_async`]. params: Box, }, /// A shared template read from template cache, to be assembled on the way out. @@ -2253,8 +2224,8 @@ fn apply_publisher_asset_cache_policy( Ok(()) } -/// Owned version of [`ProcessResponseParams`] for returning from -/// [`handle_publisher_request`] without lifetime issues. +/// Owned body-processing parameters returned from [`handle_publisher_request`] +/// so the body can be processed after the handler returns. pub struct OwnedProcessResponseParams { /// Where to store the transformed template, or [`None`] to store nothing. /// @@ -2290,6 +2261,9 @@ pub struct OwnedProcessResponseParams { /// The origin's status, which decides whether the entries' middleware /// run on a body that is not HTML. See [`text_entry_media_type`]. pub(crate) status: StatusCode, + /// Whether the origin marked its response as one reader's, see + /// [`MiddlewareContext::origin_private`](crate::middleware::MiddlewareContext::origin_private). + pub(crate) origin_private: bool, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: AdBidsState, /// Observation context for the in-flight auction. @@ -2535,7 +2509,9 @@ pub async fn buffer_publisher_response_async( settings, integration_registry, &mut response, - )?; + services, + ) + .await?; encode_complete_body(bytes, response_compression(&response))? } else { bytes @@ -2591,7 +2567,9 @@ pub async fn buffer_publisher_response_async( settings, integration_registry, &mut response, - )?; + services, + ) + .await?; let assembled = encode_complete_body(assembled, response_compression(&response))?; response.headers_mut().insert( http::header::CONTENT_LENGTH, @@ -2803,6 +2781,8 @@ fn build_template_assembly_params( request_path: String::new(), content_type: entry.metadata.content_type.clone(), status: StatusCode::OK, + // A stored template passed the gate that keeps one reader's page out. + origin_private: false, // The template already carries the head seam; re-injecting would duplicate it. permissions_json, ad_slots_script: None, @@ -3158,7 +3138,7 @@ pub async fn publisher_response_into_streaming_response( // Each piece goes through the serve entry as it leaves, so // the head is still written before the auction is awaited. let encoded_head = encoder - .encode_chunk(serve_chunk(&mut serve, head.to_vec(), false)?) + .encode_chunk(serve_chunk(&mut serve, head.to_vec())?) .map_err(publisher_stream_error)?; if !encoded_head.is_empty() { yield bytes::Bytes::from(encoded_head); @@ -3189,14 +3169,14 @@ pub async fn publisher_response_into_streaming_response( let seam = seam_script_for(¶ms); if !seam.is_empty() { let encoded_seam = encoder - .encode_chunk(serve_chunk(&mut serve, seam.into_bytes(), false)?) + .encode_chunk(serve_chunk(&mut serve, seam.into_bytes())?) .map_err(publisher_stream_error)?; if !encoded_seam.is_empty() { yield bytes::Bytes::from(encoded_seam); } } let encoded_tail = encoder - .encode_chunk(serve_chunk(&mut serve, tail.to_vec(), true)?) + .encode_chunk(serve_last_chunk(&mut serve, tail.to_vec(), &services).await?) .map_err(publisher_stream_error)?; if !encoded_tail.is_empty() { yield bytes::Bytes::from(encoded_tail); @@ -3410,9 +3390,14 @@ pub async fn publisher_response_into_streaming_response( yield encoded; } } - for encoded in - passthrough_finish_segments(&mut processor, &mut decoder, &mut encoder) - .map_err(publisher_stream_error)? + for encoded in passthrough_finish_segments( + &mut processor, + &mut decoder, + &mut encoder, + &services, + ) + .await + .map_err(publisher_stream_error)? { yield encoded; } @@ -3571,55 +3556,6 @@ impl Write for BoundedWriter { } } -/// Stream the publisher response body through the processing pipeline. -/// -/// Called by the adapter after `stream_to_client()` has committed the response -/// headers. Runs synchronously against an already-materialised body; the async -/// I/O happened upstream in [`handle_publisher_request`]. Writes processed -/// chunks directly to `output`. -/// -/// # Errors -/// -/// Returns an error if processing fails mid-stream. Since headers are already -/// committed, the caller should log the error and drop the `StreamingBody` -/// (client sees a truncated response). -pub fn stream_publisher_body( - body: EdgeBody, - output: &mut W, - params: &OwnedProcessResponseParams, - settings: &Settings, - integration_registry: &IntegrationRegistry, -) -> Result, Report> { - let permissions_script = permissions_script_for(params, settings); - let borrowed = ProcessResponseParams { - content_encoding: ¶ms.content_encoding, - origin_host: ¶ms.origin_host, - origin_url: ¶ms.origin_url, - request_host: ¶ms.request_host, - request_scheme: ¶ms.request_scheme, - request_path: ¶ms.request_path, - settings, - content_type: ¶ms.content_type, - status: params.status, - integration_registry, - permissions_script: permissions_script.as_deref(), - ad_slots_script: params.ad_slots_script.as_deref(), - ad_bids_state: params.ad_bids_state.script_cell(), - request_state: ¶ms.request_state, - shared_template_authorized: params.template_cache_key.is_some(), - csp_nonce_observed: params.csp_nonce_observed.as_ref(), - prepared: ¶ms.prepared, - public_prefix: ¶ms.public_prefix, - }; - let input_compression = Compression::from_content_encoding(¶ms.content_encoding); - let output_compression = if params.template_cache_key.is_some() { - Compression::None - } else { - input_compression - }; - process_response_streaming(body, output, &borrowed, output_compression) -} - /// Stream publisher body with parser-confirmed live bid injection. /// /// Drives the origin body through the HTML pipeline one chunk at a time. The @@ -3632,9 +3568,11 @@ pub fn stream_publisher_body( /// 3. The generated marker is replaced with the collected bid script. /// /// For non-HTML content types the auction is collected before any body bytes -/// are written (no `` to inject). If `params.dispatched_auction` is -/// `None` the function falls back to the synchronous -/// [`stream_publisher_body`] path. +/// are written (no `` to inject). Without a dispatched auction the body +/// takes the same chunk pipeline with no seam, whether it was read whole or +/// arrives as a stream. Every path ends the processor with its asynchronous +/// finish, so a middleware processor's asynchronous work completes before the +/// output is finalized. /// /// # Errors /// @@ -3650,19 +3588,15 @@ pub async fn stream_publisher_body_async( services: &RuntimeServices, ) -> Result, Report> { let Some(dispatched) = params.dispatched_auction.take() else { - if body.is_stream() { - return process_response_streaming_async( - body, - output, - params, - settings, - integration_registry, - ) - .await; - } - - // No auction and already-buffered body — keep the existing sync pipeline. - return stream_publisher_body(body, output, params, settings, integration_registry); + return process_response_streaming_async( + body, + output, + params, + settings, + integration_registry, + services, + ) + .await; }; let telemetry = AuctionTelemetryCarry { observation: params.auction_observation.take(), @@ -3683,17 +3617,15 @@ pub async fn stream_publisher_body_async( settings, ) .await; - if body.is_stream() { - return process_response_streaming_async( - body, - output, - params, - settings, - integration_registry, - ) - .await; - } - return stream_publisher_body(body, output, params, settings, integration_registry); + return process_response_streaming_async( + body, + output, + params, + settings, + integration_registry, + services, + ) + .await; } // HTML: let lol_html mark the structural body end for the auction seam. @@ -3708,6 +3640,8 @@ pub async fn stream_publisher_body_async( prepared: Arc::clone(¶ms.prepared), public_prefix: ¶ms.public_prefix, origin_host: ¶ms.origin_host, + status: params.status, + origin_private: params.origin_private, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, request_path: ¶ms.request_path, @@ -3913,6 +3847,8 @@ fn write_serve_middleware_headers( request_host: reader.request_host, request_scheme: reader.request_scheme, origin_host: reader.origin_host, + status: response.status(), + origin_private: reader.origin_private, platform: reader.platform, document_state: &document_state, max_buffered_script_bytes: reader.max_buffered_script_bytes, @@ -4848,11 +4784,12 @@ async fn body_close_hold_loop( loop { match reader.read(&mut buffer) { Ok(0) => { - let final_out = match processor.process_chunk(&[], true).change_context( - TrustedServerError::Proxy { + let final_out = match processor + .finish(&[], deps.services) + .await + .change_context(TrustedServerError::Proxy { message: "Failed to finalize processor".to_string(), - }, - ) { + }) { Ok(output) => output, Err(err) => { abandon_reader_auction( @@ -6121,6 +6058,7 @@ pub async fn handle_publisher_request( let document = TextResponse { content_type: crate::middleware::HTML_MEDIA_TYPE, status: StatusCode::OK, + origin_private: false, request_path: &request_path, request_host, request_scheme, @@ -6280,6 +6218,11 @@ pub async fn handle_publisher_request( response = empty_document(&entry.media_type)?; } + // Read before core writes its own cache policy on the response below, so a + // middleware is told what the origin declared. + let origin_private = response.headers().contains_key(header::SET_COOKIE) + || cache_control_forbids_shared_storage(response.headers()); + // Every serve middleware an entry names for this path is asked for the // headers it sets on the response that is sent, whatever it is, so a // redirect with no body reaches a middleware that rewrites its Location. @@ -6288,6 +6231,8 @@ pub async fn handle_publisher_request( request_host, request_scheme, origin_host: &origin_host_header, + status: response.status(), + origin_private, platform: integration_registry.platform(), request_state: &request_state, max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, @@ -6538,6 +6483,7 @@ pub async fn handle_publisher_request( let document = TextResponse { content_type: &content_type, status, + origin_private, request_path: &request_path, request_host, request_scheme, @@ -6677,6 +6623,7 @@ pub async fn handle_publisher_request( request_path: request_path.clone(), content_type, status, + origin_private, permissions_json, ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), @@ -9821,6 +9768,26 @@ mod tests { use http::{Method, Request as HttpRequest, StatusCode, header}; use std::sync::Arc; + /// Runs the body pipeline with no auction over `body`, ending the + /// processor with the asynchronous finish on services with no timer, as a + /// buffered adapter does with a body it read whole. + fn stream_publisher_body( + body: EdgeBody, + output: &mut W, + params: &OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, + ) -> Result, Report> { + futures::executor::block_on(process_response_streaming_async( + body, + output, + params, + settings, + integration_registry, + &noop_services(), + )) + } + /// A resolved permission state as page JSON, for tests that need a payload /// with something in it rather than the empty state. /// @@ -11368,6 +11335,7 @@ mod tests { request_path: "/".to_owned(), content_type: "application/json".to_owned(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -11828,6 +11796,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -23795,6 +23765,7 @@ mod tests { request_path: "/a.txt".to_owned(), content_type: content_type.to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -23911,6 +23882,7 @@ mod tests { request_path: "/a.txt".to_owned(), content_type: fixture::FRAMED_MEDIA_TYPE.to_string(), status: StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -23980,6 +23952,7 @@ mod tests { request_path: "/a.txt".to_owned(), content_type: "text/plain; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -24035,6 +24008,7 @@ mod tests { request_path: "/a.txt".to_owned(), content_type: content_type.to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -26760,6 +26734,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -26826,6 +26801,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -26913,6 +26889,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27009,6 +26986,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27120,6 +27098,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27172,6 +27151,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27304,6 +27284,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27374,6 +27355,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27447,6 +27429,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27520,6 +27503,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27593,6 +27577,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27646,6 +27631,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27875,6 +27861,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -27956,6 +27943,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -28039,6 +28027,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -28116,6 +28105,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -28275,6 +28265,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -28955,6 +28946,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -29156,6 +29148,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -29241,6 +29234,7 @@ mod tests { request_path: "/".to_owned(), content_type: "Text/HTML; Charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -29309,6 +29303,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -29432,6 +29427,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -29495,6 +29491,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index 4c751ebdcd..1b379ec8c3 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -54,7 +54,7 @@ pub trait StreamProcessor { /// Reset the processor state (useful for reuse) fn reset(&mut self) {} - /// Start an optional asynchronous finish of this processor's final output. + /// Starts an optional asynchronous finish of this processor's final output. /// /// `held` is exactly what the final [`process_chunk`](Self::process_chunk) /// call (`is_last = true`) returned: the bytes the processor emits when it @@ -62,15 +62,16 @@ pub trait StreamProcessor { /// once, right after that final call, with the request's /// [`RuntimeServices`]. /// - /// Returning `None` (the default) emits `held` unchanged; no future is - /// created. Returning a finish makes the driver await it and emit its + /// Returning `None`, the default, emits `held` unchanged and creates no + /// future. Returning a finish makes the driver await it and emit its /// output in place of `held`. The output is infallible: a processor absorbs /// its own failures and returns the bytes to emit, typically `held` itself. /// On a runtime with a [`PlatformTimer`](crate::platform::PlatformTimer), /// the driver drops the finish once the deadline given to /// [`AsyncFinish::new`] elapses and emits `held` instead. /// - /// Only drivers that can await call this. A synchronous driver, such as + /// Only a driver that can await calls this, through + /// [`finish`](Self::finish). A synchronous driver, such as /// [`StreamingPipeline`], emits `held`. fn async_finish<'a>( &'a mut self, @@ -80,6 +81,30 @@ pub trait StreamProcessor { let _ = (held, services); None } + + /// Ends the document with `chunk` as its last input, awaiting any + /// asynchronous finish, for a driver that can await. Called once, in place + /// of `process_chunk(chunk, true)`. + /// + /// The default processes `chunk` as the last chunk and runs + /// [`async_finish`](Self::async_finish) over what that returned. A + /// processor made of others ends each of them in order with their own + /// `finish`, so each is handed what the one before it finished with, and + /// none is handed its last chunk before the one before it has finished. + /// + /// # Errors + /// + /// Returns an error if processing the last chunk fails. + fn finish<'a>( + &'a mut self, + chunk: &'a [u8], + services: &'a RuntimeServices, + ) -> LocalBoxFuture<'a, Result, io::Error>> { + Box::pin(async move { + let held = self.process_chunk(chunk, true)?; + Ok(finish_held(self, held, services).await) + }) + } } /// Asynchronous work a [`StreamProcessor`] completes after its final chunk. @@ -92,7 +117,7 @@ pub struct AsyncFinish<'a> { } impl<'a> AsyncFinish<'a> { - /// Wrap `output`, which resolves to the bytes to emit, with the time the + /// Wraps `output`, which resolves to the bytes to emit, with the time the /// driver may wait for it. /// /// `deadline` is measured from when the driver starts awaiting the finish. @@ -106,14 +131,15 @@ impl<'a> AsyncFinish<'a> { } } -/// Run `processor`'s optional asynchronous finish over `held`, its final output. +/// Runs `processor`'s optional asynchronous finish over `held`, its final +/// output. /// /// Returns the finish output, or `held` when the processor has no finish. When /// `services` has a timer, the finish races its deadline; on expiry the finish /// future is dropped and `held` is returned. Without a timer the finish is /// awaited to completion. -pub(crate) async fn finish_held( - processor: &mut dyn StreamProcessor, +async fn finish_held( + processor: &mut P, held: Vec, services: &RuntimeServices, ) -> Vec { @@ -130,7 +156,7 @@ pub(crate) async fn finish_held( } }; log::warn!( - "stream processor finish exceeded its {} ms deadline; emitting its held output", + "A stream processor's finish ran past its {} ms deadline, so its held output is emitted", deadline.as_millis() ); held From 6ddcfaf2a936d50d25fa1c5f3c46548e5709ee4d Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:09:37 +0000 Subject: [PATCH 04/15] Wire Axum runtime KV from TRUSTED_SERVER_KV_PATH --- Cargo.lock | 3 + crates/trusted-server-adapter-axum/Cargo.toml | 3 + .../trusted-server-adapter-axum/src/main.rs | 19 +- .../src/platform.rs | 112 ++++++++-- .../tests/runtime_kv.rs | 205 ++++++++++++++++++ docs/guide/axum-dev.md | 10 + 6 files changed, 334 insertions(+), 18 deletions(-) create mode 100644 crates/trusted-server-adapter-axum/tests/runtime_kv.rs diff --git a/Cargo.lock b/Cargo.lock index 89550f2f36..90fda973c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6594,9 +6594,11 @@ dependencies = [ name = "trusted-server-adapter-axum" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "axum", "base64", + "bytes", "edgezero-adapter-axum", "edgezero-core", "error-stack", @@ -6606,6 +6608,7 @@ dependencies = [ "serde_json", "simple_logger", "temp-env", + "tempfile", "tokio", "tower 0.4.13", "trusted-server-ad-server-iabtl", diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index 47604fc941..02ca020e30 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -19,7 +19,9 @@ name = "trusted-server-axum" path = "src/main.rs" [dependencies] +anyhow = { workspace = true } async-trait = { workspace = true } +bytes = { workspace = true } edgezero-adapter-axum = { workspace = true, features = ["axum"] } edgezero-core = { workspace = true } error-stack = { workspace = true } @@ -64,6 +66,7 @@ axum = { workspace = true } base64 = { workspace = true } serde_json = { workspace = true } temp-env = { workspace = true } +tempfile = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } tower = { workspace = true, features = ["util"] } # Test fixture only. It proves the integration seam from outside core and must diff --git a/crates/trusted-server-adapter-axum/src/main.rs b/crates/trusted-server-adapter-axum/src/main.rs index 66d592e0b7..0a88091dda 100644 --- a/crates/trusted-server-adapter-axum/src/main.rs +++ b/crates/trusted-server-adapter-axum/src/main.rs @@ -1,7 +1,10 @@ +use std::path::Path; + use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; use edgezero_core::app::Hooks as _; use log::LevelFilter; use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_adapter_axum::platform::{KV_PATH_ENV, open_kv_handle}; /// Environment variable that sets the maximum log level. const LOG_LEVEL_ENV: &str = "RUST_LOG"; @@ -39,7 +42,21 @@ fn main() { log::info!("Listening on http://{}", config.addr); let router = TrustedServerApp::routes(); - if let Err(err) = AxumDevServer::with_config(router, config).run() { + let mut server = AxumDevServer::with_config(router, config); + if let Some(path) = std::env::var_os(KV_PATH_ENV).filter(|path| !path.is_empty()) { + // Construct the store once so every request shares one handle. + match open_kv_handle(Path::new(&path)) { + Ok(handle) => { + log::info!("Runtime KV: {}", Path::new(&path).display()); + server = server.with_kv_handle(handle); + } + Err(err) => { + log::error!("{KV_PATH_ENV} is set but the KV store cannot be opened: {err}"); + std::process::exit(1); + } + } + } + if let Err(err) = server.run() { log::error!("trusted-server-adapter-axum failed: {err}"); std::process::exit(1); } diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 8d0edfdc67..86c3ecbf02 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -1,18 +1,23 @@ use std::future::Future; use std::net::IpAddr; +use std::path::Path; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; use std::time::Duration; use async_trait::async_trait; +use bytes::Bytes; +use edgezero_adapter_axum::key_value_store::PersistentKvStore; use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, header}; +use edgezero_core::key_value_store::{KvHandle, KvPage, KvStore}; use error_stack::{Report, ResultExt as _}; use trusted_server_core::platform::{ - AuctionTargetCapabilities, AuctionTargetDescriptor, ClientInfo, GeoInfo, PlatformBackend, - PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, - PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, - PlatformSelectResult, RuntimeServices, StoreId, StoreName, + AuctionTargetCapabilities, AuctionTargetDescriptor, ClientInfo, GeoInfo, KvError, + PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, + PlatformHttpClient, PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, + PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, + StoreName, UnavailableKvStore, }; // --------------------------------------------------------------------------- @@ -563,17 +568,85 @@ impl PlatformHttpClient for AxumPlatformHttpClient { } } +// --------------------------------------------------------------------------- +// Runtime KV +// --------------------------------------------------------------------------- + +/// Environment variable naming the file that backs runtime KV. +/// +/// When set at startup, the dev server persists runtime KV in that file. When +/// unset, runtime KV is unavailable. +pub const KV_PATH_ENV: &str = "TRUSTED_SERVER_KV_PATH"; + +/// Open the persistent runtime KV store at `path`, creating missing parent +/// directories. +/// +/// # Errors +/// +/// Returns an error when the directory cannot be created or the database +/// cannot be opened (corrupted file, locked by another process). +pub fn open_kv_handle(path: &Path) -> Result { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent).map_err(|err| { + KvError::Internal(anyhow::anyhow!( + "failed to create KV directory `{}`: {err}", + parent.display() + )) + })?; + } + Ok(KvHandle::new(Arc::new(PersistentKvStore::new(path)?))) +} + +/// Bridges edgezero's [`KvHandle`] to [`PlatformKvStore`]. +struct KvHandleAdapter(KvHandle); + +#[async_trait(?Send)] +impl KvStore for KvHandleAdapter { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + self.0.get_bytes(key).await + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.0.put_bytes(key, value).await + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + self.0.put_bytes_with_ttl(key, value, ttl).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.0.delete(key).await + } + + async fn list_keys_page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + self.0.list_keys_page(prefix, cursor, limit).await + } +} + // --------------------------------------------------------------------------- // build_runtime_services // --------------------------------------------------------------------------- /// Construct [`RuntimeServices`] for an incoming Axum request. /// -/// # Degraded features in dev +/// # Runtime KV /// -/// The generic runtime KV slot uses -/// [`trusted_server_core::platform::UnavailableKvStore`]. A `warn` log is -/// emitted once per process. +/// Runtime KV comes from the KV handle the dev server injects into each +/// request, which `main` builds when [`KV_PATH_ENV`] is set. Without it the +/// slot is [`UnavailableKvStore`] and a `warn` log is emitted once per process. pub fn build_runtime_services( ctx: &edgezero_core::context::RequestContext, settings: &trusted_server_core::settings::Settings, @@ -581,17 +654,25 @@ pub fn build_runtime_services( [Arc], >, ) -> RuntimeServices { - static KV_WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); - KV_WARNED.get_or_init(|| { - log::warn!("Axum dev server: generic runtime KV is unavailable (UnavailableKvStore)."); - }); + let kv_store: Arc = match ctx.kv_store_default() { + Some(handle) => Arc::new(KvHandleAdapter(handle)), + None => { + static KV_WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); + KV_WARNED.get_or_init(|| { + log::warn!( + "Axum dev server: generic runtime KV is unavailable; set {KV_PATH_ENV} to enable it." + ); + }); + Arc::new(UnavailableKvStore) + } + }; let client_ip = edgezero_adapter_axum::context::AxumRequestContext::get(ctx.request()) .and_then(|c| c.remote_addr) .map(|addr| addr.ip()); use trusted_server_core::platform::{ - PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformKvStore, PlatformSecretStore, + PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore, }; // Stateless shims are promoted to process-wide statics so callers clone @@ -600,7 +681,6 @@ pub fn build_runtime_services( std::sync::OnceLock::new(); static SECRET_STORE: std::sync::OnceLock> = std::sync::OnceLock::new(); - static KV_STORE: std::sync::OnceLock> = std::sync::OnceLock::new(); static BACKEND: std::sync::OnceLock> = std::sync::OnceLock::new(); static GEO: std::sync::OnceLock> = std::sync::OnceLock::new(); @@ -611,9 +691,7 @@ pub fn build_runtime_services( .secret_store(Arc::clone(SECRET_STORE.get_or_init(|| { Arc::new(AxumPlatformSecretStore) as Arc }))) - .kv_store(Arc::clone(KV_STORE.get_or_init(|| { - Arc::new(trusted_server_core::platform::UnavailableKvStore) as Arc - }))) + .kv_store(kv_store) .backend(Arc::clone(BACKEND.get_or_init(|| { Arc::new(AxumPlatformBackend) as Arc }))) diff --git a/crates/trusted-server-adapter-axum/tests/runtime_kv.rs b/crates/trusted-server-adapter-axum/tests/runtime_kv.rs new file mode 100644 index 0000000000..9be72ff12d --- /dev/null +++ b/crates/trusted-server-adapter-axum/tests/runtime_kv.rs @@ -0,0 +1,205 @@ +//! Runtime KV wiring for the Axum dev server. +//! +//! Each test builds the KV handle exactly as `main` does (`open_kv_handle` on +//! the `TRUSTED_SERVER_KV_PATH` value) and attaches it with +//! `EdgeZeroAxumService::with_kv_handle`, the same call the dev server makes. +//! A probe route reads and writes through `build_runtime_services`, so the +//! adapter's own request-to-`RuntimeServices` path is what is exercised. + +use axum::body::{Body as AxumBody, to_bytes}; +use axum::http::Request; +use bytes::Bytes; +use edgezero_adapter_axum::service::EdgeZeroAxumService; +use edgezero_core::body::Body; +use edgezero_core::context::RequestContext; +use edgezero_core::http::{HandlerFuture, StatusCode, response_builder}; +use edgezero_core::router::RouterService; +use tower::{Service as _, ServiceExt as _}; +use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_adapter_axum::platform::{build_runtime_services, open_kv_handle}; + +const PROBE_KEY: &str = "probe-key"; + +fn reply( + status: StatusCode, + body: Body, +) -> Result { + Ok(response_builder() + .status(status) + .body(body) + .expect("should build probe response")) +} + +/// The services the request path builds for `ctx`, with default settings and +/// no permission signal module. +fn services_for(ctx: &RequestContext) -> trusted_server_core::platform::RuntimeServices { + build_runtime_services( + ctx, + &trusted_server_core::settings::Settings::default(), + &std::sync::Arc::default(), + ) +} + +fn probe_router() -> RouterService { + RouterService::builder() + .put("/probe", |ctx: RequestContext| -> HandlerFuture { + Box::pin(async move { + let services = services_for(&ctx); + match services + .kv_store() + .put_bytes(PROBE_KEY, Bytes::from_static(b"stored")) + .await + { + Ok(()) => reply(StatusCode::NO_CONTENT, Body::empty()), + Err(_) => reply(StatusCode::SERVICE_UNAVAILABLE, Body::empty()), + } + }) + }) + .get("/probe", |ctx: RequestContext| -> HandlerFuture { + Box::pin(async move { + let services = services_for(&ctx); + match services.kv_store().get_bytes(PROBE_KEY).await { + Ok(Some(value)) => reply(StatusCode::OK, Body::from(value.to_vec())), + Ok(None) => reply(StatusCode::NOT_FOUND, Body::empty()), + Err(_) => reply(StatusCode::SERVICE_UNAVAILABLE, Body::empty()), + } + }) + }) + .build() +} + +async fn call(service: &mut EdgeZeroAxumService, method: &str) -> (u16, Bytes) { + let request = Request::builder() + .method(method) + .uri("/probe") + .body(AxumBody::empty()) + .expect("should build probe request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should serve probe request"); + let status = response.status().as_u16(); + let body = to_bytes(response.into_body(), 1024) + .await + .expect("should read probe body"); + (status, body) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn runtime_kv_persists_across_requests_and_restarts() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let path = dir.path().join("nested").join("kv.redb"); + + { + let handle = open_kv_handle(&path).expect("should open KV and create its directory"); + let mut service = EdgeZeroAxumService::new(probe_router()).with_kv_handle(handle); + assert_eq!(call(&mut service, "PUT").await.0, 204, "should write"); + let (status, body) = call(&mut service, "GET").await; + assert_eq!(status, 200, "a second request should see the write"); + assert_eq!(body, Bytes::from_static(b"stored")); + } + + let handle = open_kv_handle(&path).expect("should reopen the same KV file"); + let mut restarted = EdgeZeroAxumService::new(probe_router()).with_kv_handle(handle); + let (status, body) = call(&mut restarted, "GET").await; + assert_eq!(status, 200, "data should survive a process restart"); + assert_eq!(body, Bytes::from_static(b"stored")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn runtime_kv_is_isolated_between_server_instances() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let mut first = EdgeZeroAxumService::new(probe_router()) + .with_kv_handle(open_kv_handle(&dir.path().join("first.redb")).expect("should open")); + let mut second = EdgeZeroAxumService::new(probe_router()) + .with_kv_handle(open_kv_handle(&dir.path().join("second.redb")).expect("should open")); + + assert_eq!( + call(&mut first, "PUT").await.0, + 204, + "should write to first" + ); + + assert_eq!( + call(&mut second, "GET").await.0, + 404, + "a write to one instance must not be visible to another" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn runtime_kv_is_unavailable_without_a_configured_path() { + let mut service = EdgeZeroAxumService::new(probe_router()); + + assert_eq!( + call(&mut service, "GET").await.0, + 503, + "with no KV handle injected, KV should report unavailable" + ); +} + +#[test] +fn open_kv_handle_rejects_a_path_that_cannot_be_created() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let blocker = dir.path().join("file"); + std::fs::write(&blocker, b"not a directory").expect("should write blocker file"); + + let result = open_kv_handle(&blocker.join("kv.redb")); + + assert!(result.is_err(), "a path under a regular file should fail"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn application_serves_requests_with_and_without_runtime_kv() { + let settings = trusted_server_core::settings::Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "integration-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + "#, + ) + .expect("should parse settings"); + let dir = tempfile::tempdir().expect("should create temp dir"); + let handle = open_kv_handle(&dir.path().join("kv.redb")).expect("should open KV"); + + for kv in [None, Some(handle)] { + let router = + TrustedServerApp::routes_with_settings(settings.clone()).expect("should build router"); + let mut service = EdgeZeroAxumService::new(router); + if let Some(handle) = kv { + service = service.with_kv_handle(handle); + } + let request = Request::builder() + .method("POST") + .uri("/_ts/admin/keys/rotate") + .body(AxumBody::empty()) + .expect("should build request"); + + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should serve request"); + + assert_eq!( + response.status().as_u16(), + 401, + "the app should answer the same with or without runtime KV" + ); + } +} diff --git a/docs/guide/axum-dev.md b/docs/guide/axum-dev.md index 73988b96b8..d7ce5d8802 100644 --- a/docs/guide/axum-dev.md +++ b/docs/guide/axum-dev.md @@ -59,6 +59,16 @@ its `ext.debug` echo of the resolved request) when the `[demand.]` table of a Prebid Server demand source sets `debug = true`. Enable it only for short local debugging sessions. +## Runtime KV + +Generic runtime KV (`RuntimeServices::kv_store`) is off by default. Set +`TRUSTED_SERVER_KV_PATH` to a file path before starting the server and the dev +server opens one persistent store at that path, creating missing directories, +and shares it across all requests. Data survives restarts; two servers pointed +at different paths do not see each other's data. If the variable is set but the +store cannot be opened, the server exits with an error. When it is unset, +runtime KV reports unavailable and a warning is logged once. + ## Runtime boundaries Axum sanitizes forwarded client-IP headers but does not resolve From 47afad3562346c7e726cbaa0c62601b2da788f09 Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:09:55 +0000 Subject: [PATCH 05/15] Run Axum runtime KV operations on the blocking pool --- .../src/platform.rs | 166 +++++++++++++++++- 1 file changed, 165 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 86c3ecbf02..0afbe89000 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -597,7 +597,88 @@ pub fn open_kv_handle(path: &Path) -> Result { )) })?; } - Ok(KvHandle::new(Arc::new(PersistentKvStore::new(path)?))) + Ok(KvHandle::new(Arc::new(BlockingPoolKvStore(Arc::new( + PersistentKvStore::new(path)?, + ))))) +} + +/// Runs each operation of a synchronous [`KvStore`] on Tokio's blocking pool. +/// +/// [`PersistentKvStore`] does its `redb` transactions inside its `async fn`s +/// without yielding, so a slow commit would hold the request task and keep +/// any timer raced against it (an integration deadline, for example) from +/// firing. Here the request task only awaits the blocking task and stays +/// free to observe its deadline. +/// +/// A caller that stops waiting drops only the join handle: the operation +/// keeps running to its end in the background. That is safe for `redb`. A +/// write either commits atomically or fails and its transaction aborts on +/// drop, and `redb` admits one write transaction at a time, so a later +/// operation waits for the abandoned one instead of interleaving with it. +/// Dropping the Tokio runtime waits for running blocking tasks, so a commit in +/// flight is not cut short when the server stops. +struct BlockingPoolKvStore(Arc); + +impl BlockingPoolKvStore { + async fn run(&self, op: F) -> Result + where + T: Send + 'static, + F: FnOnce(Arc) -> Fut + Send + 'static, + Fut: Future>, + { + let store = Arc::clone(&self.0); + tokio::task::spawn_blocking(move || futures::executor::block_on(op(store))) + .await + .map_err(|err| KvError::Internal(anyhow::anyhow!("KV task failed: {err}")))? + } +} + +#[async_trait(?Send)] +impl KvStore for BlockingPoolKvStore { + async fn delete(&self, key: &str) -> Result<(), KvError> { + let key = key.to_owned(); + self.run(move |store| async move { store.delete(&key).await }) + .await + } + + async fn get_bytes(&self, key: &str) -> Result, KvError> { + let key = key.to_owned(); + self.run(move |store| async move { store.get_bytes(&key).await }) + .await + } + + async fn list_keys_page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + let prefix = prefix.to_owned(); + let cursor = cursor.map(str::to_owned); + self.run(move |store| async move { + store + .list_keys_page(&prefix, cursor.as_deref(), limit) + .await + }) + .await + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + let key = key.to_owned(); + self.run(move |store| async move { store.put_bytes(&key, value).await }) + .await + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + let key = key.to_owned(); + self.run(move |store| async move { store.put_bytes_with_ttl(&key, value, ttl).await }) + .await + } } /// Bridges edgezero's [`KvHandle`] to [`PlatformKvStore`]. @@ -1072,6 +1153,89 @@ mod tests { ); } + /// Stands in for a slow `redb` commit: `put_bytes` blocks its thread + /// until the test releases it. + struct GatedStore { + gate: std::sync::Mutex>, + finished: std::sync::atomic::AtomicBool, + } + + #[async_trait(?Send)] + impl KvStore for GatedStore { + async fn delete(&self, _key: &str) -> Result<(), KvError> { + Err(KvError::Unavailable) + } + + async fn get_bytes(&self, _key: &str) -> Result, KvError> { + Err(KvError::Unavailable) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Err(KvError::Unavailable) + } + + async fn put_bytes(&self, _key: &str, _value: Bytes) -> Result<(), KvError> { + let _ = self + .gate + .lock() + .expect("should lock gate") + .recv_timeout(Duration::from_secs(5)); + self.finished + .store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + _key: &str, + _value: Bytes, + _ttl: Duration, + ) -> Result<(), KvError> { + Err(KvError::Unavailable) + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn blocking_kv_op_does_not_delay_a_racing_deadline() { + let (release, gate) = std::sync::mpsc::channel(); + let store = Arc::new(GatedStore { + gate: std::sync::Mutex::new(gate), + finished: std::sync::atomic::AtomicBool::new(false), + }); + let handle = KvHandle::new(Arc::new(BlockingPoolKvStore(Arc::clone(&store)))); + + let started = std::time::Instant::now(); + let raced = tokio::time::timeout( + Duration::from_millis(50), + handle.put_bytes("key", Bytes::from_static(b"value")), + ) + .await; + let elapsed = started.elapsed(); + + assert!( + raced.is_err(), + "the deadline should fire while the store op is still blocked" + ); + assert!( + elapsed < Duration::from_secs(2), + "the deadline should fire on time, fired after {elapsed:?}" + ); + + release.send(()).expect("should release the blocked op"); + tokio::time::timeout(Duration::from_secs(5), async { + while !store.finished.load(std::sync::atomic::Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the abandoned op should still run to completion"); + } + async fn serve_raw_response(response: &'static [u8]) -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await From d5f250fbdcb25e07d719028f5f7c80341aa943d8 Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:10:32 +0000 Subject: [PATCH 06/15] Provide platform timers on Cloudflare, Spin and Axum --- .../src/platform.rs | 39 ++++++++++++++++++- .../src/platform.rs | 34 +++++++++++++++- .../src/platform.rs | 31 ++++++++++++++- 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 0afbe89000..b97eeef211 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -16,8 +16,8 @@ use trusted_server_core::platform::{ AuctionTargetCapabilities, AuctionTargetDescriptor, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, - PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, - StoreName, UnavailableKvStore, + PlatformResponse, PlatformSecretStore, PlatformSelectResult, PlatformTimer, RuntimeServices, + StoreId, StoreName, UnavailableKvStore, }; // --------------------------------------------------------------------------- @@ -764,6 +764,7 @@ pub fn build_runtime_services( std::sync::OnceLock::new(); static BACKEND: std::sync::OnceLock> = std::sync::OnceLock::new(); static GEO: std::sync::OnceLock> = std::sync::OnceLock::new(); + static TIMER: std::sync::OnceLock> = std::sync::OnceLock::new(); RuntimeServices::builder() .config_store(Arc::clone(CONFIG_STORE.get_or_init(|| { @@ -791,6 +792,9 @@ pub fn build_runtime_services( // crates this adapter links, so every request asks exactly the ones // configuration named, in that order. .permission_signal_modules(Arc::clone(permission_signal_modules)) + .timer(Some(Arc::clone(TIMER.get_or_init(|| { + Arc::new(AxumPlatformTimer) as Arc + })))) .client_info(ClientInfo { client_ip, tls_protocol: None, @@ -800,6 +804,20 @@ pub fn build_runtime_services( .build() } +// --------------------------------------------------------------------------- +// AxumPlatformTimer +// --------------------------------------------------------------------------- + +/// Wall-clock timer backed by the tokio runtime's time driver. +pub struct AxumPlatformTimer; + +#[async_trait(?Send)] +impl PlatformTimer for AxumPlatformTimer { + async fn sleep(&self, duration: Duration) { + tokio::time::sleep(duration).await; + } +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -850,6 +868,23 @@ mod tests { ); } + #[tokio::test] + async fn runtime_timer_releases_work_that_never_completes() { + let services = test_services(); + let timer = services.timer().expect("should provide a runtime timer"); + + let raced = futures::future::select( + Box::pin(futures::future::pending::<()>()), + timer.sleep(Duration::from_millis(5)), + ) + .await; + + assert!( + matches!(raced, futures::future::Either::Right(_)), + "should fire while the other future stays pending" + ); + } + #[test] fn the_backend_name_and_transport_timer_are_pinned() { let backend = AxumPlatformBackend; diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 8cbb0a93c0..d8c387ec02 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -11,8 +11,8 @@ use error_stack::ResultExt as _; use trusted_server_core::platform::{ AuctionTargetCapabilities, AuctionTargetDescriptor, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, - PlatformHttpClient, PlatformKvStore, PlatformSecretStore, RuntimeServices, StoreId, StoreName, - UnavailableKvStore, + PlatformHttpClient, PlatformKvStore, PlatformSecretStore, PlatformTimer, RuntimeServices, + StoreId, StoreName, UnavailableKvStore, }; #[cfg(not(target_arch = "wasm32"))] @@ -614,6 +614,35 @@ impl PlatformSecretStore for CloudflareSecretStoreAdapter { } } +// --------------------------------------------------------------------------- +// Timer — wasm32 only (Workers `setTimeout`) +// --------------------------------------------------------------------------- + +/// Wall-clock timer backed by the Workers `setTimeout`. +/// +/// workers-rs [`worker::Delay`] clears its timeout when dropped, so a +/// cancelled sleep leaves no timer behind in the isolate. +#[cfg(target_arch = "wasm32")] +struct CloudflareTimer; + +#[cfg(target_arch = "wasm32")] +#[async_trait::async_trait(?Send)] +impl PlatformTimer for CloudflareTimer { + async fn sleep(&self, duration: Duration) { + worker::Delay::from(duration).await; + } +} + +/// The Workers timer on wasm32; none on the native host, which has no +/// Workers event loop. +fn runtime_timer() -> Option> { + #[cfg(target_arch = "wasm32")] + let timer: Option> = Some(Arc::new(CloudflareTimer)); + #[cfg(not(target_arch = "wasm32"))] + let timer: Option> = None; + timer +} + // --------------------------------------------------------------------------- // build_runtime_services // --------------------------------------------------------------------------- @@ -693,6 +722,7 @@ pub fn build_runtime_services( // crates this adapter links, so every request asks exactly the ones // configuration named, in that order. .permission_signal_modules(Arc::clone(permission_signal_modules)) + .timer(runtime_timer()) .client_info(ClientInfo { client_ip, tls_protocol: None, diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 9a3a160a36..f9efe3f96d 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -13,8 +13,8 @@ use http_body_util::BodyExt as _; use trusted_server_core::platform::{ AuctionTargetCapabilities, AuctionTargetDescriptor, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, - PlatformHttpClient, PlatformKvStore, PlatformSecretStore, RuntimeServices, StoreId, StoreName, - UnavailableKvStore, + PlatformHttpClient, PlatformKvStore, PlatformSecretStore, PlatformTimer, RuntimeServices, + StoreId, StoreName, UnavailableKvStore, }; #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] @@ -757,6 +757,32 @@ impl PlatformSecretStore for SpinSecretStoreAdapter { } } +// --------------------------------------------------------------------------- +// Timer — Spin wasm32 only (WASI monotonic clock) +// --------------------------------------------------------------------------- + +/// Wall-clock timer backed by the WASI monotonic clock through +/// [`spin_sdk::time::sleep`]. +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +struct SpinPlatformTimer; + +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +#[async_trait::async_trait(?Send)] +impl PlatformTimer for SpinPlatformTimer { + async fn sleep(&self, duration: Duration) { + spin_sdk::time::sleep(duration).await; + } +} + +/// The WASI timer in the Spin component; none on the native host. +fn runtime_timer() -> Option> { + #[cfg(all(feature = "spin", target_arch = "wasm32"))] + let timer: Option> = Some(Arc::new(SpinPlatformTimer)); + #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] + let timer: Option> = None; + timer +} + // --------------------------------------------------------------------------- // build_runtime_services // --------------------------------------------------------------------------- @@ -816,6 +842,7 @@ pub fn build_runtime_services( // crates this adapter links, so every request asks exactly the ones // configuration named, in that order. .permission_signal_modules(Arc::clone(permission_signal_modules)) + .timer(runtime_timer()) .client_info(ClientInfo { client_ip, tls_protocol: None, From a0eb046ce90bf0693146668ccb14c600963f293d Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:18:51 +0000 Subject: [PATCH 07/15] Move Encypher text provenance into the provenance.encypher module crate The Encypher signing integration becomes a serve middleware in crates/provenance/encypher, selected by [provenance] module = "encypher" and placed by a [[serve]] entry. Its preparation applies the request gates with no I/O, and its stream processor signs the article in the asynchronous finish core now awaits. The worker token is a module secret setting. The enabled flag and path_prefixes give way to the selection and the entry's path, and the module is offered on every adapter and the CLI. --- Cargo.lock | 26 + Cargo.toml | 2 + crates/provenance/encypher/Cargo.toml | 39 + crates/provenance/encypher/src/api.rs | 303 ++ crates/provenance/encypher/src/boundary.rs | 272 ++ crates/provenance/encypher/src/embed.rs | 477 ++++ .../encypher/src/fixtures/parity.json | 848 ++++++ crates/provenance/encypher/src/lib.rs | 346 +++ crates/provenance/encypher/src/processor.rs | 2520 +++++++++++++++++ crates/provenance/encypher/src/tests.rs | 399 +++ crates/provenance/encypher/src/text.rs | 500 ++++ crates/trusted-server-adapter-axum/Cargo.toml | 1 + crates/trusted-server-adapter-axum/src/app.rs | 1 + .../Cargo.toml | 1 + .../src/app.rs | 1 + .../trusted-server-adapter-fastly/Cargo.toml | 1 + .../trusted-server-adapter-fastly/src/app.rs | 1 + crates/trusted-server-adapter-spin/Cargo.toml | 1 + crates/trusted-server-adapter-spin/src/app.rs | 1 + crates/trusted-server-cli/Cargo.toml | 1 + crates/trusted-server-cli/src/app_config.rs | 1 + crates/trusted-server-core/src/publisher.rs | 93 +- 22 files changed, 5830 insertions(+), 5 deletions(-) create mode 100644 crates/provenance/encypher/Cargo.toml create mode 100644 crates/provenance/encypher/src/api.rs create mode 100644 crates/provenance/encypher/src/boundary.rs create mode 100644 crates/provenance/encypher/src/embed.rs create mode 100644 crates/provenance/encypher/src/fixtures/parity.json create mode 100644 crates/provenance/encypher/src/lib.rs create mode 100644 crates/provenance/encypher/src/processor.rs create mode 100644 crates/provenance/encypher/src/tests.rs create mode 100644 crates/provenance/encypher/src/text.rs diff --git a/Cargo.lock b/Cargo.lock index 90fda973c2..a9586ea2e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6636,6 +6636,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6686,6 +6687,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6742,6 +6744,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6794,6 +6797,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -7020,6 +7024,7 @@ dependencies = [ "trusted-server-identity-lockr", "trusted-server-loading-cache-51degrees", "trusted-server-middleware-51degrees", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -7352,6 +7357,27 @@ dependencies = [ "trusted-server-permission-signal-gpc", ] +[[package]] +name = "trusted-server-provenance-encypher" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "edgezero-core", + "error-stack", + "futures", + "http", + "log", + "regex", + "serde", + "serde_json", + "sha2 0.10.9", + "trusted-server-core", + "url", + "validator", + "web-time", +] + [[package]] name = "trusted-server-shared-51degrees" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4f1e312212..1f27b93efb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "crates/ads-txt/iabtl", "crates/analytics/tinybird", "crates/headers/geo", + "crates/provenance/encypher", "crates/loading-cache/51degrees", "crates/shared/aws", "crates/shared/51degrees", @@ -176,6 +177,7 @@ trusted-server-cmp-inmobi = { path = "crates/cmp/inmobi" } trusted-server-ads-txt-iabtl = { path = "crates/ads-txt/iabtl" } trusted-server-analytics-tinybird = { path = "crates/analytics/tinybird" } trusted-server-headers-geo = { path = "crates/headers/geo" } +trusted-server-provenance-encypher = { path = "crates/provenance/encypher" } trusted-server-loading-cache-51degrees = { path = "crates/loading-cache/51degrees" } trusted-server-shared-aws = { path = "crates/shared/aws" } trusted-server-shared-51degrees = { path = "crates/shared/51degrees" } diff --git a/crates/provenance/encypher/Cargo.toml b/crates/provenance/encypher/Cargo.toml new file mode 100644 index 0000000000..5c596e1370 --- /dev/null +++ b/crates/provenance/encypher/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "trusted-server-provenance-encypher" +description = "Encypher text provenance: signs the article on each page a [[serve]] entry names it for, embedding invisible markers and a C2PA manifest link, and passes every page it cannot sign through unchanged." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +bytes = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +log = { workspace = true } +regex = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } +validator = { workspace = true } +web-time = { workspace = true } + +[dev-dependencies] +async-trait = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate. +[package.metadata.maintainers] +owner = "Encypher" +status = "vendor owned" diff --git a/crates/provenance/encypher/src/api.rs b/crates/provenance/encypher/src/api.rs new file mode 100644 index 0000000000..ddf89b92a7 --- /dev/null +++ b/crates/provenance/encypher/src/api.rs @@ -0,0 +1,303 @@ +use serde::{Deserialize, Serialize}; + +/// A 2 MiB cap accommodates roughly 600 KiB surrogate-escaped plans at the 50,000-scalar input limit. +pub const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; +/// Maximum visible-text length sent for signing, measured in Unicode scalars. +pub const MAX_TEXT_SCALARS: usize = 50_000; +/// Policy responses contain only one digest; cap them independently of plans. +pub const MAX_POLICY_RESPONSE_BYTES: usize = 1024; + +/// JSON request body for the Encypher CDN signing endpoint. +#[derive(Serialize)] +pub struct SignRequest<'a> { + /// Canonical visible article text. + pub text: &'a str, + /// Absolute page URL without its query string. + pub page_url: &'a str, + /// Optional page title capped at 500 Unicode scalars. + #[serde(skip_serializing_if = "Option::is_none")] + pub document_title: Option<&'a str>, + /// Boundary detection label used for the article. + pub boundary_selector: &'a str, + /// Server policy used to select this article's cache namespace. + pub expected_policy_digest: &'a str, +} + +/// One invisible-marker insertion returned by the signing endpoint. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PlanOperation { + /// Visible codepoint index after which to insert, or -1 for before the first. + pub insert_after_index: i64, + /// Invisible marker characters to insert. + pub marker: String, +} + +/// A complete marker embedding plan. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EmbeddingPlan { + /// Index unit, which must be `codepoint`. + pub index_unit: String, + /// Ordered marker insertion operations. + pub operations: Vec, +} + +/// Parsed response from the Encypher CDN signing endpoint. +#[derive(Debug, Deserialize)] +pub struct SignResponse { + /// Whether signing succeeded. + pub success: bool, + /// Marker plan when signing succeeded. + #[serde(default)] + pub embedding_plan: Option, + /// C2PA manifest URL when signing succeeded. + #[serde(default)] + pub manifest_url: Option, + /// Stable failure reason when signing did not succeed. + #[serde(default)] + pub error: Option, + /// Actual policy used by the server; required before a successful plan is used. + #[serde(default)] + pub policy_digest: Option, +} + +/// Fail-closed signing-response parse errors. +#[derive(Debug, PartialEq)] +pub enum ApiError { + /// The response exceeded [`MAX_RESPONSE_BYTES`]. + TooLarge, + /// The response was not valid JSON for [`SignResponse`]. + Malformed, +} + +/// Parses a bounded signing response. +/// +/// # Errors +/// +/// Returns [`ApiError::TooLarge`] above the response limit and +/// [`ApiError::Malformed`] when JSON decoding fails. +pub fn parse_sign_response(body: &[u8]) -> Result { + if body.len() > MAX_RESPONSE_BYTES { + return Err(ApiError::TooLarge); + } + serde_json::from_slice(body).map_err(|_| ApiError::Malformed) +} + +/// A policy digest is exactly a lowercase SHA-256 hex value. +pub fn valid_policy_digest(digest: &str) -> bool { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Parse the policy endpoint's strict, bounded response. +/// +/// # Errors +/// +/// Rejects oversized JSON, extra or duplicate fields, and invalid digests. +pub fn parse_policy_response(body: &[u8]) -> Result { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct PolicyResponse { + policy_digest: String, + } + if body.len() > MAX_POLICY_RESPONSE_BYTES { + return Err(ApiError::TooLarge); + } + let response: PolicyResponse = serde_json::from_slice(body).map_err(|_| ApiError::Malformed)?; + if !valid_policy_digest(&response.policy_digest) { + return Err(ApiError::Malformed); + } + Ok(response.policy_digest) +} + +/// Reasons an embedding plan is unsafe or cannot address the visible text. +#[derive(Debug, PartialEq)] +pub enum PlanError { + /// The plan does not use Unicode codepoint indices. + IndexUnit, + /// An insertion index is outside the visible-text range. + IndexOutOfRange, + /// A marker is empty. + EmptyMarker, + /// A marker contains a character outside the edge-core invisible set. + DisallowedMarkerChar, + /// The plan has no operations. + Empty, + /// The manifest URL is present but not an absolute `https` URL. + ManifestUrl, +} + +/// Validates a signing result before use: index bounds and the invisible marker +/// alphabet for every operation, and a manifest URL that is absent or `https`. +/// +/// This is the only check between the API or KV and the page, so callers must +/// pass the scalar count of the exact text that was signed. +/// +/// # Errors +/// +/// Returns a [`PlanError`] for the first invalid property. +pub fn validate_plan( + plan: &EmbeddingPlan, + manifest_url: Option<&str>, + visible_scalars: usize, +) -> Result<(), PlanError> { + if plan.index_unit != "codepoint" { + return Err(PlanError::IndexUnit); + } + if plan.operations.is_empty() { + return Err(PlanError::Empty); + } + for operation in &plan.operations { + if operation.insert_after_index < -1 { + return Err(PlanError::IndexOutOfRange); + } + if operation.insert_after_index >= 0 { + let Ok(index) = usize::try_from(operation.insert_after_index) else { + return Err(PlanError::IndexOutOfRange); + }; + if index >= visible_scalars { + return Err(PlanError::IndexOutOfRange); + } + } + if operation.marker.is_empty() { + return Err(PlanError::EmptyMarker); + } + if !operation.marker.chars().all(is_allowed_marker_char) { + return Err(PlanError::DisallowedMarkerChar); + } + } + if let Some(url) = manifest_url { + let https = url::Url::parse(url).is_ok_and(|url| { + url.scheme() == "https" && url.host_str().is_some_and(|h| !h.is_empty()) + }); + if !https { + return Err(PlanError::ManifestUrl); + } + } + Ok(()) +} + +fn is_allowed_marker_char(ch: char) -> bool { + matches!(ch, '\u{fe00}'..='\u{fe0f}' | '\u{e0100}'..='\u{e01ef}' | '\u{feff}') +} + +#[cfg(test)] +mod tests { + use super::*; + + fn single_op_plan() -> EmbeddingPlan { + EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_string(), + }], + } + } + + #[test] + fn policy_response_requires_only_a_lowercase_sha256_digest() { + let digest = "ab".repeat(32); + let body = format!(r#"{{"policy_digest":"{digest}"}}"#); + assert_eq!(parse_policy_response(body.as_bytes()), Ok(digest.clone())); + for body in [ + "{}".to_owned(), + r#"{"policy_digest":null}"#.to_owned(), + r#"{"policy_digest":42}"#.to_owned(), + format!(r#"{{"policy_digest":"{}"}}"#, digest.to_uppercase()), + format!(r#"{{"policy_digest":"{}"}}"#, "g".repeat(64)), + format!(r#"{{"policy_digest":"{}"}}"#, "a".repeat(63)), + format!(r#"{{"policy_digest":"{digest}","extra":true}}"#), + format!(r#"{{"policy_digest":"{digest}","policy_digest":"{digest}"}}"#), + format!(r#"{{"policy_digest":"{digest}"}} trailing"#), + ] { + assert_eq!( + parse_policy_response(body.as_bytes()), + Err(ApiError::Malformed) + ); + } + assert_eq!( + parse_policy_response(&vec![b' '; MAX_POLICY_RESPONSE_BYTES + 1]), + Err(ApiError::TooLarge) + ); + } + + #[test] + fn parses_and_validates_a_response_plan() { + let body = r#"{"success":true,"embedding_plan":{"index_unit":"codepoint","operations":[{"insert_after_index":0,"marker":"︀"}]},"manifest_url":"https://manifest.example.com/x"}"#; + let response = parse_sign_response(body.as_bytes()).expect("should parse"); + let plan = response.embedding_plan.expect("should contain plan"); + assert_eq!( + validate_plan(&plan, response.manifest_url.as_deref(), 1), + Ok(()) + ); + } + + #[test] + fn manifest_url_must_be_absent_or_https() { + let plan = single_op_plan(); + assert_eq!(validate_plan(&plan, None, 1), Ok(())); + for url in [ + "", + "http://manifest.example.com/x", + "javascript:alert(1)", + "data:text/html,", + "/api/v1/public/cdn/manifest/1", + "https://", + ] { + assert_eq!( + validate_plan(&plan, Some(url), 1), + Err(PlanError::ManifestUrl), + "{url} should be rejected" + ); + } + } + + #[test] + fn rejects_markup_and_bad_ranges() { + let markup = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "".to_string(), + }], + }; + assert_eq!( + validate_plan(&markup, None, 1), + Err(PlanError::DisallowedMarkerChar) + ); + assert_eq!( + validate_plan(&single_op_plan(), None, 0), + Err(PlanError::IndexOutOfRange) + ); + } + + #[test] + fn parses_realistic_large_marker_response() { + let escaped_marker = r"\uDB40\uDD00".repeat(50_000); + let body = format!( + r#"{{"success":true,"embedding_plan":{{"index_unit":"codepoint","operations":[{{"insert_after_index":0,"marker":"{escaped_marker}"}}]}}}}"# + ); + assert!(body.len() > 600_000); + let response = + parse_sign_response(body.as_bytes()).expect("large valid response should parse"); + let plan = response + .embedding_plan + .expect("response should contain a plan"); + assert_eq!(plan.operations[0].marker.chars().count(), 50_000); + } + + #[test] + fn enforces_size_and_empty_plan_limits() { + assert_eq!( + parse_sign_response(&vec![b' '; MAX_RESPONSE_BYTES + 1]).map(|_| ()), + Err(ApiError::TooLarge) + ); + let empty = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: Vec::new(), + }; + assert_eq!(validate_plan(&empty, None, 1), Err(PlanError::Empty)); + } +} diff --git a/crates/provenance/encypher/src/boundary.rs b/crates/provenance/encypher/src/boundary.rs new file mode 100644 index 0000000000..b2b78c749d --- /dev/null +++ b/crates/provenance/encypher/src/boundary.rs @@ -0,0 +1,272 @@ +use std::sync::LazyLock; + +use regex::Regex; + +const CMS_CONTENT_CLASSES: [&str; 13] = [ + "entry-content", + "wp-block-post-content", + "post-content", + "article-body", + "article-content", + "story-body", + "story-body-text", + "gh-content", + "w-richtext", + "sqs-block-content", + "body markup", + "e-content", + "RichTextStoryBody", +]; + +static ARTICLE: LazyLock> = LazyLock::new(|| tag_opening("article")); +static MAIN: LazyLock> = LazyLock::new(|| tag_opening("main")); +static BODY: LazyLock> = LazyLock::new(|| tag_opening("body")); +static CLUSTER_CONTAINERS: LazyLock> = LazyLock::new(|| { + ["div", "section", "main"] + .into_iter() + .filter_map(tag_opening) + .collect() +}); +static PARAGRAPH: LazyLock> = LazyLock::new(|| Regex::new(r"(?i)> = LazyLock::new(|| { + CMS_CONTENT_CLASSES + .into_iter() + .filter_map(|class_name| Some((class_name, class_opening(class_name)?))) + .collect() +}); +static ITEMPROP_ARTICLE_BODY: LazyLock> = + LazyLock::new(|| attribute_opening("itemprop", "articleBody")); +static ROLE_MAIN: LazyLock> = LazyLock::new(|| attribute_opening("role", "main")); +static SELECTOR_ATTRIBUTE: LazyLock> = + LazyLock::new(|| Regex::new(r#"\[([A-Za-z0-9_]+)=["']?([^"'\]]+)["']?\]"#).ok()); + +/// The UTF-8 byte range and detection label of an article's outer HTML. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Boundary { + /// Inclusive byte offset of the opening element. + pub start: usize, + /// Exclusive byte offset after the first matching closing element. + pub end: usize, + /// Detection-chain selector label. + pub selector: String, +} + +/// Finds an article boundary using the edge-core priority chain, excluding JSON-LD P5. +#[must_use] +pub fn find_article_boundary(html: &str, override_selector: Option<&str>) -> Option { + if let Some(selector) = override_selector.filter(|selector| !selector.is_empty()) + && let Some(range) = match_selector(html, selector) + { + return Some(Boundary { + start: range.0, + end: range.1, + selector: format!("override:{selector}"), + }); + } + + let articles = ARTICLE + .as_ref() + .map_or_else(Vec::new, |opening| match_all(html, opening)); + if let Some(first) = articles.first().copied() { + let mut selected = first; + let mut selected_count = count_paragraphs(html.get(first.0..first.1).unwrap_or("")); + for candidate in articles.iter().copied().skip(1) { + let candidate_count = + count_paragraphs(html.get(candidate.0..candidate.1).unwrap_or("")); + if candidate_count > selected_count { + selected = candidate; + selected_count = candidate_count; + } + } + return Some(boundary(selected, "article")); + } + + for (class_name, opening) in CMS_CLASSES.iter() { + if let Some(range) = match_first(html, opening) { + return Some(boundary(range, &format!("class:{class_name}"))); + } + } + + if let Some(range) = match_fixed(html, &ITEMPROP_ARTICLE_BODY) { + return Some(boundary(range, "itemprop:articleBody")); + } + if let Some(range) = match_fixed(html, &MAIN) { + return Some(boundary(range, "main")); + } + if let Some(range) = match_fixed(html, &ROLE_MAIN) { + return Some(boundary(range, "role:main")); + } + if let Some(range) = find_largest_p_cluster(html) { + return Some(boundary(range, "p-cluster")); + } + match_fixed(html, &BODY).map(|range| boundary(range, "body")) +} + +fn boundary((start, end): (usize, usize), selector: &str) -> Boundary { + Boundary { + start, + end, + selector: selector.to_owned(), + } +} + +// Every opening pattern captures the element's tag name as group 1. +fn tag_opening(tag_name: &str) -> Option { + Regex::new(&format!( + r"(?i)<({})(?-u:\b)[^>]*>", + regex::escape(tag_name) + )) + .ok() +} + +fn class_opening(class_name: &str) -> Option { + Regex::new(&format!( + r#"(?i)<([A-Za-z0-9_]+)(?-u:\b)[^>]*class\s*=\s*["'][^"']*(?-u:\b){}(?-u:\b)[^"']*["'][^>]*>"#, + regex::escape(class_name) + )) + .ok() +} + +fn attribute_opening(attribute: &str, value: &str) -> Option { + Regex::new(&format!( + r#"(?i)<([A-Za-z0-9_]+)(?-u:\b)[^>]*{}\s*=\s*["']{}["'][^>]*>"#, + regex::escape(attribute), + regex::escape(value) + )) + .ok() +} + +fn match_fixed(html: &str, opening: &LazyLock>) -> Option<(usize, usize)> { + match_first(html, opening.as_ref()?) +} + +fn match_first(html: &str, opening: &Regex) -> Option<(usize, usize)> { + let captures = opening.captures(html)?; + let whole = captures.get(0)?; + close_first_element(html, whole.start(), whole.end(), captures.get(1)?.as_str()) +} + +fn match_all(html: &str, opening: &Regex) -> Vec<(usize, usize)> { + let mut matches = Vec::new(); + let mut cursor = 0_usize; + while let Some(tail) = html.get(cursor..) { + let Some(captures) = opening.captures(tail) else { + break; + }; + let (Some(whole), Some(tag_name)) = (captures.get(0), captures.get(1)) else { + break; + }; + let start = cursor.saturating_add(whole.start()); + let opening_end = cursor.saturating_add(whole.end()); + let Some(range) = close_first_element(html, start, opening_end, tag_name.as_str()) else { + break; + }; + matches.push(range); + cursor = range.1; + } + matches +} + +// An operator-configured selector is compiled per document. +fn match_selector(html: &str, selector: &str) -> Option<(usize, usize)> { + if let Some(class_name) = selector.strip_prefix('.') { + return match_first(html, &class_opening(class_name)?); + } + if selector.starts_with('[') + && let Some(captures) = SELECTOR_ATTRIBUTE.as_ref()?.captures(selector) + { + let opening = attribute_opening(captures.get(1)?.as_str(), captures.get(2)?.as_str())?; + return match_first(html, &opening); + } + match_first(html, &tag_opening(selector)?) +} + +/// Extends an opening tag to the first `` after it, ASCII case-insensitively. +fn close_first_element( + html: &str, + opening_start: usize, + opening_end: usize, + tag_name: &str, +) -> Option<(usize, usize)> { + let closing = format!(""); + let tail = html.as_bytes().get(opening_end..)?; + let offset = tail + .windows(closing.len()) + .position(|window| window.eq_ignore_ascii_case(closing.as_bytes()))?; + Some(( + opening_start, + opening_end + .saturating_add(offset) + .saturating_add(closing.len()), + )) +} + +fn count_paragraphs(html: &str) -> usize { + PARAGRAPH + .as_ref() + .map_or(0, |regex| regex.find_iter(html).count()) +} + +fn find_largest_p_cluster(html: &str) -> Option<(usize, usize)> { + let mut best = None; + let mut best_count = 4_usize; + for opening in CLUSTER_CONTAINERS.iter() { + for range in match_all(html, opening) { + let count = count_paragraphs(html.get(range.0..range.1).unwrap_or("")); + if count > best_count { + best = Some(range); + best_count = count; + } + } + } + best +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn follows_priority_chain_and_byte_offsets() { + let html = "🐟

Main.

Chosen.

"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!(boundary.selector, "article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("

Chosen.

") + ); + } + + #[test] + fn nested_same_name_uses_first_closing_tag() { + let html = "
outer
inner
tail
"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("
outer
inner
") + ); + } + + #[test] + fn closing_tags_match_ascii_case_insensitively() { + let html = "

One.

x
"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("

One.

") + ); + let boundary = + find_article_boundary(html, Some(".entry-content")).expect("should find override"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("
x
") + ); + } + + #[test] + fn selects_largest_paragraph_cluster_at_five() { + let html = "

1

2

3

4

5

"; + let boundary = find_article_boundary(html, None).expect("should find cluster"); + assert_eq!(boundary.selector, "p-cluster"); + } +} diff --git a/crates/provenance/encypher/src/embed.rs b/crates/provenance/encypher/src/embed.rs new file mode 100644 index 0000000000..b5195ba58b --- /dev/null +++ b/crates/provenance/encypher/src/embed.rs @@ -0,0 +1,477 @@ +use std::sync::LazyLock; + +use regex::Regex; + +use super::api::EmbeddingPlan; +use super::boundary::Boundary; +use super::text::{ + assemble_text, decode_html_entities, extract_fragments, is_js_whitespace, structural_close_tag, +}; + +const MAGIC_PREFIX: &str = "\u{e01df}\u{e01e0}\u{e01e1}\u{e01e2}"; + +static MANIFEST_META: LazyLock> = LazyLock::new(|| { + Regex::new(r#"(?i)]*name\s*=\s*['"]c2pa-manifest-url['"][^>]*>"#).ok() +}); + +struct MappedChar { + ch: char, + start: usize, + end: usize, +} + +/// Reports whether a string contains the four-character Encypher magic prefix. +#[must_use] +pub fn has_existing_markers(s: &str) -> bool { + s.contains(MAGIC_PREFIX) +} + +/// Reports whether HTML already declares a C2PA manifest URL meta element. +#[must_use] +pub fn has_manifest_meta(html: &str) -> bool { + MANIFEST_META + .as_ref() + .is_some_and(|regex| regex.is_match(html)) +} + +/// Applies a codepoint embedding plan within an article boundary. +/// +/// The plan must already have passed [`super::api::validate_plan`]; markers are +/// inserted verbatim. Returns `None` when the boundary has no visible text, an +/// index has no text position, or text-to-byte alignment fails. +#[must_use] +pub fn apply_plan(html: &str, boundary: &Boundary, plan: &EmbeddingPlan) -> Option { + let boundary_html = html.get(boundary.start..boundary.end)?; + let fragments = extract_fragments(boundary_html); + let visible = assemble_text(boundary_html, &fragments); + let visible_chars: Vec = visible.chars().collect(); + if visible_chars.is_empty() { + return None; + } + + let mut insertion_offsets = vec![None; visible_chars.len()]; + let mut first_visible_start = None; + let mut previous_visible_end = None; + let mut visible_cursor = 0_usize; + + for fragment in fragments { + let raw = boundary_html.get(fragment.start..fragment.end)?; + let normalized = normalize_mapped(decode_with_offsets(raw)); + if normalized.is_empty() { + continue; + } + + while visible_chars.get(visible_cursor) == Some(&' ') { + if let Some(offset) = previous_visible_end + && let Some(slot) = insertion_offsets.get_mut(visible_cursor) + { + *slot = Some(offset); + } + visible_cursor = visible_cursor.saturating_add(1); + } + + for item in normalized { + let target = visible_chars.get(visible_cursor)?; + if !equivalent(item.ch, *target) { + return None; + } + let start = fragment.start.checked_add(item.start)?; + let end = fragment.start.checked_add(item.end)?; + if first_visible_start.is_none() { + first_visible_start = Some(start); + } + let slot = insertion_offsets.get_mut(visible_cursor)?; + *slot = Some(end); + previous_visible_end = Some(end); + visible_cursor = visible_cursor.saturating_add(1); + } + } + + while visible_chars.get(visible_cursor) == Some(&' ') { + if let Some(offset) = previous_visible_end { + let slot = insertion_offsets.get_mut(visible_cursor)?; + *slot = Some(offset); + } + visible_cursor = visible_cursor.saturating_add(1); + } + if visible_cursor != visible_chars.len() { + return None; + } + let first_visible_start = first_visible_start?; + + let mut insertions = Vec::with_capacity(plan.operations.len()); + let mut output_len = html.len(); + for operation in &plan.operations { + let relative = if operation.insert_after_index == -1 { + first_visible_start + } else { + let index = usize::try_from(operation.insert_after_index).ok()?; + insertion_offsets.get(index).copied().flatten()? + }; + let absolute = boundary.start.checked_add(relative)?; + output_len = output_len.checked_add(operation.marker.len())?; + insertions.push((absolute, operation.marker.as_str())); + } + // Stable sorting preserves API order for multiple markers at the same offset. + insertions.sort_by_key(|(offset, _)| *offset); + let mut result = String::with_capacity(output_len); + let mut cursor = 0; + for (offset, marker) in insertions { + result.push_str(html.get(cursor..offset)?); + result.push_str(marker); + cursor = offset; + } + result.push_str(html.get(cursor..)?); + Some(result) +} + +/// Injects an escaped manifest meta element before the structural closing head tag. +/// +/// Returns the input unchanged when the URL is empty, a manifest meta already exists, +/// or no structural closing head tag exists. +#[must_use] +pub fn inject_manifest_meta(html: &str, manifest_url: &str) -> String { + if html.is_empty() || manifest_url.is_empty() || has_manifest_meta(html) { + return html.to_owned(); + } + let Some(offset) = structural_close_tag(html, "head") else { + return html.to_owned(); + }; + let safe_url = manifest_url + .replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">"); + let meta = format!(r#""#); + let Some(before) = html.get(..offset) else { + return html.to_owned(); + }; + let Some(after) = html.get(offset..) else { + return html.to_owned(); + }; + let mut output = String::with_capacity(html.len().saturating_add(meta.len())); + output.push_str(before); + output.push_str(&meta); + output.push_str(after); + output +} + +fn decode_with_offsets(raw: &str) -> Vec { + let mut mapped = Vec::new(); + let mut cursor = 0_usize; + while cursor < raw.len() { + if raw.as_bytes().get(cursor) == Some(&b'&') + && let Some(relative_semicolon) = raw + .as_bytes() + .get(cursor.saturating_add(1)..) + // Entity names/numbers cannot contain another ampersand or markup. + // Stop there rather than rescanning the suffix for every literal '&'. + .and_then(|tail| { + let end = tail + .iter() + .position(|byte| !byte.is_ascii_alphanumeric() && *byte != b'#')?; + (tail[end] == b';').then_some(end) + }) + { + let end = cursor + .saturating_add(1) + .saturating_add(relative_semicolon) + .saturating_add(1); + if let Some(token) = raw.get(cursor..end) { + let decoded = decode_html_entities(token); + if decoded != token { + mapped.extend(decoded.chars().map(|ch| MappedChar { + ch, + start: cursor, + end, + })); + cursor = end; + continue; + } + } + } + + let Some(ch) = raw.get(cursor..).and_then(|tail| tail.chars().next()) else { + break; + }; + let end = cursor.saturating_add(ch.len_utf8()); + mapped.push(MappedChar { + ch, + start: cursor, + end, + }); + cursor = end; + } + mapped +} + +fn normalize_mapped(mapped: Vec) -> Vec { + let mut normalized: Vec = Vec::with_capacity(mapped.len()); + for item in mapped { + if is_js_whitespace(item.ch) { + let Some(last) = normalized.last_mut() else { + continue; + }; + if last.ch == ' ' { + last.end = item.end; + } else { + normalized.push(MappedChar { + ch: ' ', + start: item.start, + end: item.end, + }); + } + } else { + normalized.push(item); + } + } + while normalized.last().is_some_and(|item| item.ch == ' ') { + normalized.pop(); + } + normalized +} + +fn equivalent(left: char, right: char) -> bool { + left == right || matches!((left, right), ('\u{00a0}', ' ') | (' ', '\u{00a0}')) +} + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + use super::*; + use crate::api::{EmbeddingPlan, PlanOperation}; + use crate::boundary::{Boundary, find_article_boundary}; + use crate::text::{assemble_text, extract_fragments}; + + #[test] + fn adversarial_ampersands_have_bounded_embedding_work() { + let html = format!("
{}
", "&".repeat(100_000)); + let boundary = find_article_boundary(&html, None).expect("article"); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_owned(), + }], + }; + let output = apply_plan(&html, &boundary, &plan).expect("valid plan"); + assert_eq!( + output, + format!("
&\u{fe00}{}
", "&".repeat(99_999)) + ); + } + + #[test] + fn dense_plan_on_large_document_preserves_operation_order() { + let html = format!( + "
{}
", + "a".repeat(50_000), + "x".repeat(4_900_000) + ); + let boundary = find_article_boundary(&html, None).expect("article"); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: (0..20_000) + .rev() + .map(|index| PlanOperation { + insert_after_index: index, + marker: "\u{fe00}".to_owned(), + }) + .collect(), + }; + let output = apply_plan(&html, &boundary, &plan).expect("valid dense plan"); + assert_eq!( + output, + format!( + "
{}{}
", + "a\u{fe00}".repeat(20_000), + "a".repeat(30_000), + "x".repeat(4_900_000) + ) + ); + } + + #[derive(Deserialize)] + struct FixtureFile { + cases: Vec, + } + + #[derive(Deserialize)] + struct FixtureCase { + id: String, + html: String, + override_selector: Option, + boundary: FixtureBoundary, + text: String, + text_rust: String, + text_divergence: Option, + plan: EmbeddingPlan, + expected_output: String, + invalid_plan: Option, + expected_invalid_output: Option, + manifest_url: String, + expected_meta_output_js: String, + expected_meta_output_rust: String, + meta_divergence: Option, + } + + #[derive(Deserialize)] + struct FixtureBoundary { + selector: String, + start: usize, + end: usize, + } + + fn fixtures() -> FixtureFile { + serde_json::from_str(include_str!("fixtures/parity.json")) + .expect("parity fixtures should deserialize") + } + + #[test] + fn groups_same_offset_in_plan_order() { + let html = "🐟
A B
"; + let start = html.find("
").expect("should find boundary"); + let boundary = Boundary { + start, + end: html.len(), + selector: "article".to_string(), + }; + let plan = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![ + PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_string(), + }, + PlanOperation { + insert_after_index: 0, + marker: "\u{fe01}".to_string(), + }, + ], + }; + assert_eq!( + apply_plan(html, &boundary, &plan), + Some("🐟
A\u{fe00}\u{fe01} B
".to_string()) + ); + } + + #[test] + fn manifest_meta_uses_structural_head_close_and_escapes() { + let html = ""; + let output = inject_manifest_meta(html, "https://example.com/?a=1&b=\"x\""); + assert!(output.contains(""#; + let html = format!("{prefix}
Visible
"); + assert_eq!( + inject_manifest_meta(&html, "javascript:alert(1)"), + format!( + "{prefix}
Visible
" + ) + ); + // The URL is inert metadata, not an href, script source or refresh target. + let script = + r#""#; + assert_eq!( + inject_manifest_meta( + &format!("{script}"), + "data:text/html," + ), + format!( + "{script}" + ) + ); + } + + #[test] + fn manifest_meta_ignores_quotes_inside_unquoted_values() { + // The browser reads `x"y` as an unquoted value, so the second meta's quoted + // content, including its ``, is attribute text. + let prefix = + r#""#; + let html = format!("{prefix}Visible"); + assert_eq!( + inject_manifest_meta(&html, "https://manifest.example.com/m"), + format!( + "{prefix}Visible" + ) + ); + } + + #[test] + fn edge_core_parity_fixtures_match() { + for fixture in fixtures().cases { + let boundary = + find_article_boundary(&fixture.html, fixture.override_selector.as_deref()) + .unwrap_or_else(|| panic!("{} should have a boundary", fixture.id)); + assert_eq!( + boundary.selector, fixture.boundary.selector, + "{}", + fixture.id + ); + assert_eq!(boundary.start, fixture.boundary.start, "{}", fixture.id); + assert_eq!(boundary.end, fixture.boundary.end, "{}", fixture.id); + let boundary_html = fixture + .html + .get(boundary.start..boundary.end) + .expect("fixture boundary should be valid"); + let fragments = extract_fragments(boundary_html); + let text = assemble_text(boundary_html, &fragments); + assert_eq!(text, fixture.text_rust, "{}", fixture.id); + if fixture.text_divergence.is_some() { + assert_ne!(text, fixture.text, "{}", fixture.id); + } else { + assert_eq!(text, fixture.text, "{}", fixture.id); + } + assert_eq!( + apply_plan(&fixture.html, &boundary, &fixture.plan), + Some(fixture.expected_output), + "{}", + fixture.id + ); + if let Some(invalid_plan) = fixture.invalid_plan { + assert_eq!( + apply_plan(&fixture.html, &boundary, &invalid_plan), + fixture.expected_invalid_output, + "{}", + fixture.id + ); + } + let meta = inject_manifest_meta(&fixture.html, &fixture.manifest_url); + assert_eq!(meta, fixture.expected_meta_output_rust, "{}", fixture.id); + if fixture.meta_divergence.is_some() { + assert_ne!(meta, fixture.expected_meta_output_js, "{}", fixture.id); + } else { + assert_eq!(meta, fixture.expected_meta_output_js, "{}", fixture.id); + } + } + } + + #[test] + fn every_fixture_byte_prefix_is_panic_free() { + for fixture in fixtures().cases { + for end in 0..=fixture.html.len() { + let prefix = String::from_utf8_lossy( + fixture + .html + .as_bytes() + .get(..end) + .expect("prefix range should be valid"), + ); + let fragments = extract_fragments(&prefix); + let _ = assemble_text(&prefix, &fragments); + if let Some(boundary) = + find_article_boundary(&prefix, fixture.override_selector.as_deref()) + { + let _ = apply_plan(&prefix, &boundary, &fixture.plan); + } + let _ = inject_manifest_meta(&prefix, &fixture.manifest_url); + } + } + } +} diff --git a/crates/provenance/encypher/src/fixtures/parity.json b/crates/provenance/encypher/src/fixtures/parity.json new file mode 100644 index 0000000000..8487676317 --- /dev/null +++ b/crates/provenance/encypher/src/fixtures/parity.json @@ -0,0 +1,848 @@ +{ + "generated_from": [ + "integrations/edge-core/boundary.js", + "integrations/edge-core/fragments.js", + "integrations/edge-core/embed.js" + ], + "fixed_plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "cases": [ + { + "id": "override-multibyte-prefix", + "html": "Café & News🙂前

Alpha & beta.

", + "override_selector": "[data=story]", + "boundary": { + "selector": "override:[data=story]", + "start": 78, + "end": 126 + }, + "text": "Alpha & beta.", + "text_rust": "Alpha & beta.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Café & News🙂前

A︀︁lpha & beta.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Café & News🙂前

Alpha & beta.

", + "expected_meta_output_rust": "Café & News🙂前

Alpha & beta.

", + "meta_divergence": null + }, + { + "id": "article-most-paragraphs", + "html": "Two articles

Short.

First paragraph.

Second paragraph.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 84, + "end": 159 + }, + "text": "First paragraph. Second paragraph .", + "text_rust": "First paragraph. Second paragraph .", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Two articles

Short.

F︀︁irst paragraph.

Second paragraph.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Two articles

Short.

First paragraph.

Second paragraph.

", + "expected_meta_output_rust": "Two articles

Short.

First paragraph.

Second paragraph.

", + "meta_divergence": null + }, + { + "id": "cms-class-entities", + "html": "

Fish & chips — © 🐟.

", + "override_selector": null, + "boundary": { + "selector": "class:entry-content", + "start": 25, + "end": 123 + }, + "text": "Fish & chips — © 🐟.", + "text_rust": "Fish & chips — © 🐟.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

F︀︁ish & chips — © 🐟.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Fish & chips — © 🐟.

", + "expected_meta_output_rust": "

Fish & chips — © 🐟.

", + "meta_divergence": null + }, + { + "id": "itemprop-nbsp", + "html": "

One two three.

", + "override_selector": null, + "boundary": { + "selector": "itemprop:articleBody", + "start": 25, + "end": 94 + }, + "text": "One two three.", + "text_rust": "One two three.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁ne two three.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

One two three.

", + "expected_meta_output_rust": "

One two three.

", + "meta_divergence": null + }, + { + "id": "main-inline-tags", + "html": "

Inline

Hello fictional example.com.

", + "override_selector": null, + "boundary": { + "selector": "main", + "start": 25, + "end": 105 + }, + "text": "Inline Hello fictional example.com.", + "text_rust": "Inline Hello fictional example.com.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

I︀︁nline

Hello fictional example.com.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Inline

Hello fictional example.com.

", + "expected_meta_output_rust": "

Inline

Hello fictional example.com.

", + "meta_divergence": null + }, + { + "id": "role-main", + "html": "

Role based article boundary.

", + "override_selector": null, + "boundary": { + "selector": "role:main", + "start": 25, + "end": 83 + }, + "text": "Role based article boundary.", + "text_rust": "Role based article boundary.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

R︀︁ole based article boundary.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Role based article boundary.

", + "expected_meta_output_rust": "

Role based article boundary.

", + "meta_divergence": null + }, + { + "id": "paragraph-cluster", + "html": "

One.

Two.

Three.

Four.

Five.

", + "override_selector": null, + "boundary": { + "selector": "p-cluster", + "start": 25, + "end": 109 + }, + "text": "One. Two. Three. Four. Five.", + "text_rust": "One. Two. Three. Four. Five.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁ne.

Two.

Three.

Four.

Five.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

One.

Two.

Three.

Four.

Five.

", + "expected_meta_output_rust": "

One.

Two.

Three.

Four.

Five.

", + "meta_divergence": null + }, + { + "id": "body-fallback", + "html": "

Fallback

Body article text.", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 19, + "end": 80 + }, + "text": "Fallback Body article text.", + "text_rust": "Fallback Body article text.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

F︀︁allback

Body article text.", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Fallback

Body article text.", + "expected_meta_output_rust": "

Fallback

Body article text.", + "meta_divergence": null + }, + { + "id": "nested-same-name-first-close", + "html": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 90 + }, + "text": "Outer start. Inner text.", + "text_rust": "Outer start. Inner text.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁uter start.

Inner text.

Outer tail omitted by parity.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "expected_meta_output_rust": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "meta_divergence": null + }, + { + "id": "head-close-inside-script", + "html": "Script case

Script boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 86, + "end": 131 + }, + "text": "Script boundary fallback.", + "text_rust": "Script boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Script case

S︀︁cript boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Script case

Script boundary fallback.

", + "expected_meta_output_rust": "Script case

Script boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside script text; Rust inserts before the structural closing head tag." + }, + { + "id": "head-close-inside-comment", + "html": "Comment case

Comment boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 80, + "end": 126 + }, + "text": "Comment boundary fallback.", + "text_rust": "Comment boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Comment case

C︀︁omment boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Comment case

Comment boundary fallback.

", + "expected_meta_output_rust": "Comment case

Comment boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside a comment; Rust inserts before the structural closing head tag." + }, + { + "id": "head-close-inside-title", + "html": "Headline </head> text

Title boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 55, + "end": 99 + }, + "text": "Title boundary fallback.", + "text_rust": "Title boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Headline </head> text

T︀︁itle boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Headline <meta name=\"c2pa-manifest-url\" content=\"https://manifest.example.com/c2pa?id=fictional&kind="article"\"></head> text

Title boundary fallback.

", + "expected_meta_output_rust": "Headline </head> text

Title boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside title text; Rust never searches raw-text or RCDATA content for the closing head tag." + }, + { + "id": "javascript-whitespace-set", + "html": "

Alpha 
beta\t gamma.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 77 + }, + "text": "Alpha beta gamma.", + "text_rust": "Alpha beta gamma.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

A︀︁lpha 
beta\t gamma.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Alpha 
beta\t gamma.

", + "expected_meta_output_rust": "

Alpha 
beta\t gamma.

", + "meta_divergence": null + }, + { + "id": "skip-raw-elements-and-numeric-entities", + "html": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 162 + }, + "text": "Visible A. Tail B.", + "text_rust": "Visible A. Tail B.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

V︀︁isible A.

Hidden code.
Hidden pre.

Tail B.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "expected_meta_output_rust": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "meta_divergence": null + }, + { + "id": "only-the-seven-skip-elements-are-hidden", + "html": "

Lead text.

Inline title.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 166 + }, + "text": "Lead text. Draft note. Frame fallback. Inline title. After.", + "text_rust": "Lead text. Draft note. Frame fallback. Inline title. After.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

Inline title.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

Inline title.

After.

", + "expected_meta_output_rust": "

Lead text.

Inline title.

After.

", + "meta_divergence": null + }, + { + "id": "self-closing-svg-hides-nothing", + "html": "

Lead text.

After icon.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 103 + }, + "text": "Lead text. After icon.", + "text_rust": "Lead text. After icon.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After icon.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After icon.

", + "expected_meta_output_rust": "

Lead text.

After icon.

", + "meta_divergence": null + }, + { + "id": "quoted-gt-in-attribute", + "html": "

Lead text.

\"a b\">Tail text.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 95 + }, + "text": "Lead text. b\">Tail text.", + "text_rust": "Lead text. Tail text.", + "text_divergence": "JS ends a tag at the first \">\", so the rest of the alt value becomes text; Rust ends it at the first \">\" outside a quoted attribute value.", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

\"a b\">Tail text.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

\"a b\">Tail text.

", + "expected_meta_output_rust": "

Lead text.

\"a b\">Tail text.

", + "meta_divergence": null + }, + { + "id": "nested-same-name-ignored-element", + "html": "

Lead text.

outer inner tail

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 116 + }, + "text": "Lead text. After.", + "text_rust": "Lead text. tail After.", + "text_divergence": "JS counts nested start and end tags and drops the outer tail; Rust closes the ignored region at the first matching end tag.", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

outer inner tail

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

outer inner tail

After.

", + "expected_meta_output_rust": "

Lead text.

outer inner tail

After.

", + "meta_divergence": null + }, + { + "id": "self-closing-script-stays-raw-text", + "html": "

Lead text.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 107 + }, + "text": "Lead text. var hidden = 1; After.", + "text_rust": "Lead text. After.", + "text_divergence": "JS treats \".", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After.

", + "expected_meta_output_rust": "

Lead text.

After.

", + "meta_divergence": null + }, + { + "id": "end-tag-with-attributes-closes-ignored-element", + "html": "

Lead text.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 132 + }, + "text": "Lead text. After.", + "text_rust": "Lead text. After.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After.

", + "expected_meta_output_rust": "

Lead text.

After.

", + "meta_divergence": null + }, + { + "id": "abrupt-and-bang-comment-ends", + "html": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 131 + }, + "text": "Lead text. Shown two.", + "text_rust": "Lead text. Shown one. Shown two. Shown three. After.", + "text_divergence": "JS ends every comment at the next \"-->\"; Rust, like browsers, treats \"\" and \"\" as whole comments and also ends a comment at \"--!>\".", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

Shown one.Shown two.Shown three.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "expected_meta_output_rust": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "meta_divergence": null + }, + { + "id": "out-of-range-index", + "html": "

Small boundary.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 66 + }, + "text": "Small boundary.", + "text_rust": "Small boundary.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

S︀︁mall boundary.

", + "invalid_plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": 15, + "marker": "︀" + } + ] + }, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Small boundary.

", + "expected_meta_output_rust": "

Small boundary.

", + "meta_divergence": null + } + ] +} diff --git a/crates/provenance/encypher/src/lib.rs b/crates/provenance/encypher/src/lib.rs new file mode 100644 index 0000000000..e633f6ee09 --- /dev/null +++ b/crates/provenance/encypher/src/lib.rs @@ -0,0 +1,346 @@ +//! Encypher text provenance for article pages, `provenance.encypher`. +//! +//! A serve middleware finds the article in each HTML page a `[[serve]]` entry +//! names it for, sends its visible text to the Encypher signing API, and +//! embeds the returned invisible markers and a manifest meta tag. Plans are +//! cached in the runtime KV store, bound to a short-lived server policy. +//! Every failure passes the page through unchanged. +//! +//! The middleware's preparation applies the request gates with no I/O: only +//! a `GET` of a `200` page on the configured domain, whose origin did not +//! mark it as one reader's, is held. The signing work runs in the stream +//! processor's asynchronous finish, which core awaits at the end of the +//! document. The module requires no permission: it sends the publisher's own +//! article text and page address, never anything of the reader's, and a page +//! the origin marked as one reader's is never sent. + +use std::any::Any; +use std::sync::Arc; + +use error_stack::Report; +use serde::Deserialize; +use validator::{Validate, ValidationError}; + +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationRegistration, ModuleSecretSetting, SuppliedMiddleware, +}; +use trusted_server_core::middleware::{ + HTML_MEDIA_TYPE, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, Preparation, +}; +use trusted_server_core::module_context::{FromModuleContext as _, ModuleCall, ModuleRequest}; +use trusted_server_core::redacted::Redacted; +use trusted_server_core::settings::{IntegrationConfig, Settings}; + +mod api; +mod boundary; +mod embed; +mod processor; +mod text; + +use processor::{PageRequest, Signer, document_processor, is_excluded_path}; + +/// The id the module's registration and its API backend carry. +const INTEGRATION_ID: &str = "provenance_encypher"; + +/// The name `[provenance] module` selects the module by and a `[[serve]]` +/// entry names its middleware by, from this crate's folder. +pub const MODULE: &str = "provenance.encypher"; + +/// `[provenance.encypher]`, the module's settings. +#[derive(Debug, Clone, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +#[validate(schema(function = "validate_hold_exceeds_timeout"))] +pub struct EncypherConfig { + /// Encypher API origin, for example `https://api.example.com`. Must be https. + #[validate(custom(function = validate_api_base_url))] + pub api_base_url: String, + /// Claimed publisher domain: a bare lowercase hostname without `www.`. + #[validate(custom(function = validate_domain))] + pub domain: String, + /// The name of the secret-store entry holding the `cwk_` worker + /// credential, resolved to the credential itself when settings load. + pub worker_token: Redacted, + /// Optional article selector override: `.class`, `[attr=value]` or a tag + /// name. At most 91 characters, so `override:{selector}` fits the API's + /// 100-character `boundary_selector` limit. + #[serde(default)] + #[validate(length(max = 91))] + pub article_selector: String, + /// Minimum visible article text, in Unicode scalars. + #[serde(default = "default_min_text_length")] + #[validate(range(min = 50, max = 50_000))] + pub min_text_length: usize, + /// Largest decoded document held for signing. Capped at + /// `publisher.max_buffered_body_bytes`. + #[serde(default = "default_max_document_bytes")] + #[validate(range(min = 1024))] + pub max_document_bytes: usize, + /// First-byte timeout for each policy or sign call, in milliseconds. The + /// only bound on a hung API on a runtime without a timer. + #[serde(default = "default_timeout_ms")] + #[validate(range(min = 250, max = 10_000))] + pub timeout_ms: u64, + /// Longest one document is held for signing, in milliseconds, on a + /// runtime with a timer. Must exceed `timeout_ms`. + #[serde(default = "default_max_hold_ms")] + #[validate(range(min = 500, max = 15_000))] + pub max_hold_ms: u64, + /// Lifetime of a cached signing plan, in seconds. + #[serde(default = "default_plan_ttl_seconds")] + #[validate(range(min = 60, max = 31_536_000))] + pub plan_ttl_seconds: u64, + /// Lifetime of the server-authoritative policy, in seconds. + #[serde(default = "default_policy_ttl_seconds")] + #[validate(range(min = 60, max = 300))] + pub policy_ttl_seconds: u64, + /// Lifetime of a cached signing failure, in seconds. + #[serde(default = "default_negative_ttl_seconds")] + #[validate(range(min = 60, max = 86_400))] + pub negative_ttl_seconds: u64, +} + +impl IntegrationConfig for EncypherConfig {} + +fn default_min_text_length() -> usize { + 50 +} + +fn default_max_document_bytes() -> usize { + 5_000_000 +} + +fn default_timeout_ms() -> u64 { + 3_000 +} + +fn default_max_hold_ms() -> u64 { + 5_000 +} + +fn default_plan_ttl_seconds() -> u64 { + 604_800 +} + +fn default_policy_ttl_seconds() -> u64 { + 300 +} + +fn default_negative_ttl_seconds() -> u64 { + 300 +} + +fn validate_api_base_url(value: &str) -> Result<(), ValidationError> { + let parsed = + url::Url::parse(value).map_err(|_| ValidationError::new("encypher_api_base_url"))?; + if parsed.scheme() != "https" + || parsed.host_str().is_none_or(str::is_empty) + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(ValidationError::new("encypher_api_base_url")); + } + Ok(()) +} + +fn validate_domain(value: &str) -> Result<(), ValidationError> { + let valid_label = |label: &str| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + }; + let valid = value.len() <= 253 + && value.contains('.') + && !value.starts_with("www.") + && value.split('.').all(valid_label); + if valid { + Ok(()) + } else { + Err(ValidationError::new("encypher_domain")) + } +} + +fn validate_hold_exceeds_timeout(config: &EncypherConfig) -> Result<(), ValidationError> { + if config.max_hold_ms > config.timeout_ms { + Ok(()) + } else { + Err(ValidationError::new("encypher_max_hold_ms")) + } +} + +/// The worker token, named in `[provenance.encypher]` by its entry in the +/// secret store, which is in use whenever the module is selected. +const SECRET_SETTINGS: &[ModuleSecretSetting] = &[ModuleSecretSetting { + path: &["worker_token"], + in_use: always_in_use, +}]; + +fn always_in_use(_table: &serde_json::Map) -> bool { + true +} + +/// The page change the module supplies, as a `[[serve]]` entry names it. +const SUPPLIED_MIDDLEWARE: &[SuppliedMiddleware] = + &[SuppliedMiddleware::on_html(MODULE, MiddlewarePhase::Serve)]; + +/// The builder a deployment installs to offer this module. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(INTEGRATION_ID, env!("CARGO_PKG_NAME"), register, validate) + .with_module_name(MODULE) + .with_secret_settings(SECRET_SETTINGS) + .with_middleware(SUPPLIED_MIDDLEWARE) +} + +/// Refuses a document that selects the module and names its middleware in no +/// `[[serve]]` entry, where no page would ever be signed. +fn require_an_entry(settings: &Settings) -> Result<(), Report> { + let placed = settings + .serve + .entries() + .iter() + .any(|entry| entry.middleware.iter().any(|name| name == MODULE)); + if placed { + return Ok(()); + } + Err(Report::new(TrustedServerError::Settings { + message: format!( + "[provenance] selects `encypher`, but no [[serve]] entry names `{MODULE}`, so no \ + page would be signed. Add `{MODULE}` to the [[serve]] entry that covers the \ + article pages, with a `path` for each section to sign" + ), + })) +} + +/// The module's settings when a section selects it. +fn config(settings: &Settings) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + require_an_entry(settings)?; + Ok(Some(config)) +} + +/// Validates the module's table and reports whether a section selects the +/// module. +/// +/// # Errors +/// +/// Returns an error when the table cannot be read or fails its validation, or +/// when no `[[serve]]` entry names the middleware. +pub fn validate(settings: &Settings) -> Result> { + Ok(config(settings)?.is_some()) +} + +/// Registers the middleware when a section selects the module and its worker +/// token is not blank. +/// +/// # Errors +/// +/// Returns the errors [`validate`] returns. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = config(settings)? else { + return Ok(None); + }; + if config.worker_token.expose().trim().is_empty() { + log::warn!("encypher: not registered, because the worker token is empty"); + return Ok(None); + } + log::info!("encypher: registered for domain={}", config.domain); + let max_document_bytes = config + .max_document_bytes + .min(settings.publisher.max_buffered_body_bytes); + Ok(Some( + IntegrationRegistration::builder(INTEGRATION_ID) + .without_js() + .with_middleware(Arc::new(Encypher { + signer: Arc::new(Signer::new(config, max_document_bytes)), + })) + .build(), + )) +} + +/// The middleware, which signs the article of each admitted page. +pub struct Encypher { + signer: Arc, +} + +impl std::fmt::Debug for Encypher { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("Encypher").finish_non_exhaustive() + } +} + +/// What the preparation decided for one document: the page to sign, or none. +type Admission = Option; + +impl Middleware for Encypher { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == HTML_MEDIA_TYPE + } + + /// Every path but an API, feed, admin or well-known path, or an asset, + /// which is never an article. + fn handles_path(&self, path: &str) -> bool { + !is_excluded_path(path) + } + + /// Serve only: the signed page is written on each reader's copy, so a + /// failed signing is never stored for every reader. + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + /// Applies the request gates, with no I/O, and keeps the page to sign. + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + let admission = admit(&self.signer, &call, context); + Some(Box::pin(async move { + Ok(Box::new(admission) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let Some(page) = context.take_prepared::(MODULE).flatten() else { + return MiddlewareAction::pass(); + }; + MiddlewareAction { + stream: Some(Box::new(document_processor(&self.signer, page))), + ..MiddlewareAction::pass() + } + } +} + +/// The page a document is the body of, when the gates admit it. +fn admit(signer: &Signer, call: &ModuleCall<'_>, context: &MiddlewareContext<'_>) -> Admission { + let request = ModuleRequest::from_module_context(call).ok()?; + let page = PageRequest { + method: request.method().clone(), + page_url: format!("{}://{}{}", request.scheme(), request.host(), request.path()), + response_status: context.status.as_u16(), + origin_response_is_private: context.origin_private, + }; + match signer.admit(page) { + Ok(page) => Some(page), + Err(gate) => { + log::debug!("encypher: document not held, gate={}", gate.code()); + None + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/provenance/encypher/src/processor.rs b/crates/provenance/encypher/src/processor.rs new file mode 100644 index 0000000000..c97ea37013 --- /dev/null +++ b/crates/provenance/encypher/src/processor.rs @@ -0,0 +1,2520 @@ +//! Per-document signing flow: gates, hold, KV cache, sign call, outcomes, logging. +//! +//! A document moves through four phases: +//! +//! 1. **Admit** (the middleware's preparation, with no I/O): the request +//! gates decide whether the document is held at all. +//! 2. **Extract** (synchronous): at end of input the held document's article +//! boundary, visible text and text hash are computed. +//! 3. **Resolve** (the asynchronous finish): policy and plan caches, the policy +//! and sign calls, and cache writes. On runtimes with a timer every call is +//! raced against `max_hold_ms`, keeping a reserve for the skip write. +//! 4. **Render** (synchronous): the plan's markers and the manifest meta tag are +//! applied to the held document. +//! +//! Every module failure emits the held document unchanged. + +use std::io; +use std::pin::pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use futures::future::{Either, LocalBoxFuture, select}; +use http::{HeaderValue, Method, Request, StatusCode, header}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +use crate::api::{self, ApiError, EmbeddingPlan, SignRequest}; +use crate::boundary::{self, Boundary}; +use crate::{EncypherConfig, INTEGRATION_ID, embed, text}; +use trusted_server_core::integrations::ensure_integration_backend_with_timeout; +use trusted_server_core::platform::{ + KvError, PlatformHttpRequest, PlatformKvStore, PlatformTimer, RuntimeServices, +}; +use trusted_server_core::streaming_processor::{AsyncFinish, StreamProcessor}; + +const SIGN_PATH: &str = "/api/v1/public/cdn/sign"; +const POLICY_PATH: &str = "/api/v1/public/cdn/trusted-server/policy"; +const WORKER_TOKEN_HEADER: &str = "x-encypher-worker-token"; +/// Article plan and skip keys. Entries carry their own `expires_at`. +const CACHE_KEY_PREFIX: &str = "encypher:v3"; +/// Policy digest and policy skip keys. Entries carry their own `expires_at`. +const POLICY_KEY_PREFIX: &str = "encypher:policy:v2"; +/// Part of `max_hold_ms` kept back from resolve work so a skip can still be +/// written after the work runs out of time. +const SKIP_WRITE_RESERVE: Duration = Duration::from_millis(200); +/// How far past `max_hold_ms` the core backstop deadline lies. The module +/// bounds itself first; the backstop only catches an overrun. +const BACKSTOP_MARGIN: Duration = Duration::from_millis(1_000); + +/// Path prefixes never signed (ported from the Fastly worker's `shouldSkipPath`). +const SKIP_PATH_PREFIXES: &[&str] = &[ + "/api/", + "/wp-admin/", + "/wp-json/", + "/feed/", + "/sitemap", + "/.well-known/", +]; +/// Asset suffixes never signed (ported from the Fastly worker's `shouldSkipPath`). +const SKIP_PATH_SUFFIXES: &[&str] = &[ + ".xml", ".json", ".rss", ".atom", ".css", ".js", ".woff", ".woff2", ".png", ".jpg", ".gif", + ".svg", ".ico", +]; + +/// Set once this instance has warned that the runtime KV store is unbound. +static KV_UNAVAILABLE_WARNED: AtomicBool = AtomicBool::new(false); + +/// Why a document was not held. Evaluated in the middleware's preparation, +/// with no I/O. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Gate { + Method, + Status, + OriginPrivate, + InvalidPageUrl, + HostMismatch, + ExcludedPath, +} + +impl Gate { + pub(crate) fn code(self) -> &'static str { + match self { + Self::Method => "method", + Self::Status => "status", + Self::OriginPrivate => "origin_private", + Self::InvalidPageUrl => "invalid_page_url", + Self::HostMismatch => "host_mismatch", + Self::ExcludedPath => "excluded_path", + } + } +} + +/// What the gates are told about the page a document is the body of. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PageRequest { + /// The reader's request method. + pub(crate) method: Method, + /// The scheme, host and path the reader asked for, without the query, + /// which is the address the article is signed for. + pub(crate) page_url: String, + /// The status of the response the document is the body of. + pub(crate) response_status: u16, + /// Whether the origin marked its response as one reader's. + pub(crate) origin_response_is_private: bool, +} + +/// Reason code for a held document's outcome. Sign failures are also stored +/// as the reason of a negative (`skip`) cache entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum Reason { + Signed, + CacheHit, + DocumentTooLarge, + NotUtf8, + NoBoundary, + AlreadySigned, + ManifestPresent, + TextTooShort, + KvUnavailable, + KvError, + Timeout, + CachedSkip, + CachedEntryInvalid, + CachedPlanInvalid, + RequestBuild, + Backend, + HttpError, + Forbidden, + Unprocessable, + RateLimited, + ServerError, + UnexpectedStatus, + ResponseTooLarge, + MalformedResponse, + PolicyChanged, + PolicyDigestMismatch, + QuotaExceeded, + SignRejected, + NoPlan, + InvalidPlan, + AlignmentFailed, +} + +impl Reason { + fn code(self) -> &'static str { + match self { + Self::Signed => "signed", + Self::CacheHit => "cache_hit", + Self::DocumentTooLarge => "document_too_large", + Self::NotUtf8 => "not_utf8", + Self::NoBoundary => "no_boundary", + Self::AlreadySigned => "already_signed", + Self::ManifestPresent => "manifest_present", + Self::TextTooShort => "text_too_short", + Self::KvUnavailable => "kv_unavailable", + Self::KvError => "kv_error", + Self::Timeout => "timeout", + Self::CachedSkip => "cached_skip", + Self::CachedEntryInvalid => "cached_entry_invalid", + Self::CachedPlanInvalid => "cached_plan_invalid", + Self::RequestBuild => "request_build", + Self::Backend => "backend", + Self::HttpError => "http_error", + Self::Forbidden => "forbidden", + Self::Unprocessable => "unprocessable", + Self::RateLimited => "rate_limited", + Self::ServerError => "server_error", + Self::UnexpectedStatus => "unexpected_status", + Self::ResponseTooLarge => "response_too_large", + Self::MalformedResponse => "malformed_response", + Self::PolicyChanged => "policy_changed", + Self::PolicyDigestMismatch => "policy_digest_mismatch", + Self::QuotaExceeded => "quota_exceeded", + Self::SignRejected => "sign_rejected", + Self::NoPlan => "no_plan", + Self::InvalidPlan => "invalid_plan", + Self::AlignmentFailed => "alignment_failed", + } + } +} + +/// A cache entry that records when it stops being usable. +trait Expiring { + /// Unix seconds at which the entry expires. + fn expires_at(&self) -> u64; + /// Whether the entry was written without a store TTL, so it stays until deleted. + fn ttl_less(&self) -> bool; + /// Record that the entry is written without a store TTL. + fn mark_ttl_less(&mut self); +} + +/// An article cache entry, keyed by credential context, policy and text hash. +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum CacheEntry { + Plan { + plan: EmbeddingPlan, + manifest_url: Option, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Skip { + reason: Reason, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, +} + +impl Expiring for CacheEntry { + fn expires_at(&self) -> u64 { + match self { + Self::Plan { expires_at, .. } | Self::Skip { expires_at, .. } => *expires_at, + } + } + + fn ttl_less(&self) -> bool { + match self { + Self::Plan { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less, + } + } + + fn mark_ttl_less(&mut self) { + match self { + Self::Plan { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less = true, + } + } +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum PolicyEntry { + Digest { + policy_digest: String, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Skip { + reason: Reason, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, +} + +impl Expiring for PolicyEntry { + fn expires_at(&self) -> u64 { + match self { + Self::Digest { expires_at, .. } | Self::Skip { expires_at, .. } => *expires_at, + } + } + + fn ttl_less(&self) -> bool { + match self { + Self::Digest { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less, + } + } + + fn mark_ttl_less(&mut self) { + match self { + Self::Digest { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less = true, + } + } +} + +/// The one freshness predicate applied to every cache read. +fn is_fresh(expires_at: u64, now: u64) -> bool { + now < expires_at +} + +/// Current Unix time in seconds, from a clock that works on every adapter. +fn unix_now() -> u64 { + web_time::SystemTime::now() + .duration_since(web_time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) +} + +/// Configuration shared by every document processor. +pub(crate) struct Signer { + config: EncypherConfig, + max_document_bytes: usize, + sign_url: String, + policy_url: String, + policy_key: String, + /// First 16 hex characters of `sha256(api_base_url \n domain \n token)`. + cache_context: String, + /// Set by the first TTL write the store reports as unsupported, or by reading + /// an entry written without a TTL; from then on this instance writes without + /// a TTL and deletes expired entries it reads. + ttl_less: AtomicBool, +} + +impl Signer { + /// The signer for `config`, holding at most `max_document_bytes` of one + /// document. + pub(crate) fn new(config: EncypherConfig, max_document_bytes: usize) -> Self { + let sign_url = format!("{}{SIGN_PATH}", config.api_base_url.trim_end_matches('/')); + let policy_url = format!("{}{POLICY_PATH}", config.api_base_url.trim_end_matches('/')); + let context_hash = text::content_hash_hex(&format!( + "{}\n{}\n{}", + config.api_base_url, + config.domain, + config.worker_token.expose().trim() + )); + let cache_context = context_hash + .get(..16) + .unwrap_or(context_hash.as_str()) + .to_owned(); + Self { + config, + max_document_bytes, + sign_url, + policy_url, + policy_key: format!("{POLICY_KEY_PREFIX}:{cache_context}"), + cache_context, + ttl_less: AtomicBool::new(false), + } + } +} + +/// Holds the document of an admitted page until it ends. +pub(crate) fn document_processor(signer: &Arc, request: PageRequest) -> DocumentProcessor { + DocumentProcessor { + signer: Arc::clone(signer), + request, + held: Vec::new(), + state: DocumentState::Holding, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DocumentState { + /// Accumulating the document. + Holding, + /// Over the size limit: chunks pass through and nothing is signed. + PassingThrough, + /// The whole document was emitted as held output; it may be signed in the + /// asynchronous finish. + Complete, +} + +/// Holds one admitted document until end of input, then signs it in the +/// asynchronous finish or passes it through. +pub(crate) struct DocumentProcessor { + signer: Arc, + request: PageRequest, + held: Vec, + state: DocumentState, +} + +impl StreamProcessor for DocumentProcessor { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error> { + if self.state != DocumentState::Holding { + return Ok(chunk.to_vec()); + } + if self.held.len().saturating_add(chunk.len()) > self.signer.max_document_bytes { + self.state = DocumentState::PassingThrough; + log_outcome(false, Reason::DocumentTooLarge); + let mut output = std::mem::take(&mut self.held); + output.extend_from_slice(chunk); + return Ok(output); + } + self.held.extend_from_slice(chunk); + if !is_last { + return Ok(Vec::new()); + } + // The complete document is the fail-open output; the driver owns it. + self.state = DocumentState::Complete; + Ok(std::mem::take(&mut self.held)) + } + + fn reset(&mut self) { + self.held.clear(); + self.state = DocumentState::Holding; + } + + fn async_finish<'a>( + &'a mut self, + held: &'a [u8], + services: &'a RuntimeServices, + ) -> Option> { + if self.state != DocumentState::Complete { + return None; + } + let extracted = std::str::from_utf8(held) + .map_err(|_| Reason::NotUtf8) + .and_then(|html| Ok((html, self.signer.extract(html)?))); + let (html, article) = match extracted { + Ok(extracted) => extracted, + Err(reason) => { + log_outcome(false, reason); + return None; + } + }; + let signer = &*self.signer; + let request = &self.request; + let deadline = + Duration::from_millis(signer.config.max_hold_ms).saturating_add(BACKSTOP_MARGIN); + Some(AsyncFinish::new(deadline, async move { + match signer.resolve(request, services, html, &article).await { + Ok((signed, reason)) => { + log_outcome(true, reason); + signed.into_bytes() + } + Err(reason) => { + log_outcome(false, reason); + held.to_vec() + } + } + })) + } +} + +/// The article found in a held document. +struct Article { + boundary: Boundary, + /// Visible text sent for signing, truncated to the API limit. + text: String, + /// Unicode scalars in `text`. + scalars: usize, +} + +/// Time a document's resolve may still take, on runtimes with a timer. +enum Deadline<'a> { + /// No timer: calls are bounded only by backend timeouts. + Unbounded, + /// The timer has not fired yet. + Running(LocalBoxFuture<'a, ()>), + /// The timer fired; every further race loses at once. + Expired, +} + +impl<'a> Deadline<'a> { + fn new(timer: Option<&'a dyn PlatformTimer>, budget: Duration) -> Self { + timer.map_or(Self::Unbounded, |timer| Self::Running(timer.sleep(budget))) + } + + /// Start the clock now. A timer's sleep starts on its first poll, which + /// would otherwise be the first call that does not complete at once. + async fn start(&mut self) { + if let Self::Running(sleep) = self + && futures::poll!(sleep.as_mut()).is_ready() + { + *self = Self::Expired; + } + } + + /// Run `work` to completion, or return `None` once the deadline passes. + async fn race(&mut self, work: impl Future) -> Option { + match self { + Self::Unbounded => return Some(work.await), + Self::Expired => return None, + Self::Running(sleep) => { + if let Either::Left((value, _)) = select(pin!(work), sleep.as_mut()).await { + return Some(value); + } + } + } + *self = Self::Expired; + None + } + + /// Whether the deadline has passed, so the last race lost. + fn expired(&self) -> bool { + matches!(self, Self::Expired) + } +} + +fn log_outcome(signed: bool, reason: Reason) { + if reason == Reason::KvUnavailable && !KV_UNAVAILABLE_WARNED.swap(true, Ordering::Relaxed) { + log::warn!( + "encypher: the runtime KV store is unavailable, so pages pass through unsigned; bind the adapter's KV store to enable signing" + ); + } + log::info!( + "encypher: outcome={} reason={}", + if signed { "signed" } else { "passed_through" }, + reason.code() + ); +} + +/// Whether `path` is one this module never signs: an API, feed, admin or +/// well-known path, or an asset. +pub(crate) fn is_excluded_path(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + SKIP_PATH_PREFIXES + .iter() + .any(|prefix| lower.starts_with(prefix)) + || SKIP_PATH_SUFFIXES + .iter() + .any(|suffix| lower.ends_with(suffix)) +} + +impl Signer { + /// Apply the request gates. Only an admitted document is held. + pub(crate) fn admit(&self, request: PageRequest) -> Result { + if request.method != Method::GET { + return Err(Gate::Method); + } + if request.response_status != StatusCode::OK.as_u16() { + return Err(Gate::Status); + } + if request.origin_response_is_private { + return Err(Gate::OriginPrivate); + } + let page_url = url::Url::parse(&request.page_url).map_err(|_| Gate::InvalidPageUrl)?; + let host = page_url.host_str().ok_or(Gate::InvalidPageUrl)?; + if host.strip_prefix("www.").unwrap_or(host) != self.config.domain { + return Err(Gate::HostMismatch); + } + if is_excluded_path(page_url.path()) { + return Err(Gate::ExcludedPath); + } + Ok(request) + } + + /// Find the article and its signable text, with no I/O. + fn extract(&self, html: &str) -> Result { + let boundary = boundary::find_article_boundary(html, Some(&self.config.article_selector)) + .ok_or(Reason::NoBoundary)?; + let article = html + .get(boundary.start..boundary.end) + .ok_or(Reason::NoBoundary)?; + if embed::has_existing_markers(article) { + return Err(Reason::AlreadySigned); + } + if embed::has_manifest_meta(html) { + return Err(Reason::ManifestPresent); + } + let mut visible = text::visible_text(article); + let sign_len = text::truncate_scalars(&visible, api::MAX_TEXT_SCALARS).len(); + visible.truncate(sign_len); + let scalars = text::scalar_count(&visible); + if scalars < self.config.min_text_length { + return Err(Reason::TextTooShort); + } + Ok(Article { + boundary, + text: visible, + scalars, + }) + } + + /// Unix time at which an entry written now with `ttl_seconds` expires. + fn expires_at(ttl_seconds: u64) -> u64 { + unix_now().saturating_add(ttl_seconds) + } + + /// Bound for skip writes made after resolve work stopped. + fn skip_deadline<'a>(services: &'a RuntimeServices) -> Deadline<'a> { + Deadline::new(services.timer(), SKIP_WRITE_RESERVE) + } + + /// Resolve the plan for `article` and render it into `html`. + async fn resolve( + &self, + request: &PageRequest, + services: &RuntimeServices, + html: &str, + article: &Article, + ) -> Result<(String, Reason), Reason> { + let kv = services.kv_store(); + let mut work = Deadline::new( + services.timer(), + Duration::from_millis(self.config.max_hold_ms).saturating_sub(SKIP_WRITE_RESERVE), + ); + work.start().await; + + let policy_digest = self.policy_digest(request, services, kv, &mut work).await?; + let key = format!( + "{CACHE_KEY_PREFIX}:{}:{policy_digest}:{}", + self.cache_context, + text::content_hash_hex(&article.text) + ); + let outcome = match self.read::(kv, &key, &mut work).await { + Ok(Some(CacheEntry::Plan { + plan, manifest_url, .. + })) => { + api::validate_plan(&plan, manifest_url.as_deref(), article.scalars) + .map_err(|_| Reason::CachedPlanInvalid)?; + return render(html, &article.boundary, &plan, manifest_url.as_deref()) + .map(|signed| (signed, Reason::CacheHit)) + .ok_or(Reason::CachedPlanInvalid); + } + Ok(Some(CacheEntry::Skip { .. })) => return Err(Reason::CachedSkip), + Err(Reason::Timeout) => Err(Reason::Timeout), + Err(reason) => return Err(reason), + Ok(None) => self + .request_plan(request, services, html, article, &policy_digest, &mut work) + .await + .and_then(|(plan, manifest_url)| { + api::validate_plan(&plan, manifest_url.as_deref(), article.scalars) + .map_err(|_| Reason::InvalidPlan)?; + let signed = render(html, &article.boundary, &plan, manifest_url.as_deref()) + .ok_or(Reason::AlignmentFailed)?; + Ok((signed, plan, manifest_url)) + }), + }; + match outcome { + Ok((signed, plan, manifest_url)) => { + // A plan write that runs out of time is abandoned; the page is still signed. + let entry = CacheEntry::Plan { + plan, + manifest_url, + expires_at: Self::expires_at(self.config.plan_ttl_seconds), + ttl_less: false, + }; + self.write(kv, &key, entry, self.config.plan_ttl_seconds, &mut work) + .await; + Ok((signed, Reason::Signed)) + } + Err(reason) => { + let mut deadline = Self::skip_deadline(services); + if matches!(reason, Reason::PolicyChanged | Reason::PolicyDigestMismatch) { + self.evict_policy(kv, &mut deadline).await; + } + self.write_skip(kv, &key, reason, article.scalars, &mut deadline) + .await; + Err(reason) + } + } + } + + async fn policy_digest( + &self, + request: &PageRequest, + services: &RuntimeServices, + kv: &dyn PlatformKvStore, + work: &mut Deadline<'_>, + ) -> Result { + let result = match self.read::(kv, &self.policy_key, work).await { + Ok(Some(PolicyEntry::Digest { policy_digest, .. })) + if api::valid_policy_digest(&policy_digest) => + { + return Ok(policy_digest); + } + Ok(Some(PolicyEntry::Skip { .. })) => return Err(Reason::CachedSkip), + Ok(Some(PolicyEntry::Digest { .. })) => Err(Reason::CachedEntryInvalid), + // Nothing can be cached without a store. + Err(Reason::KvUnavailable) => return Err(Reason::KvUnavailable), + Err(reason) => Err(reason), + Ok(None) => self.request_policy(request, services, work).await, + }; + let reason = match result { + Ok(policy_digest) => { + let entry = PolicyEntry::Digest { + policy_digest: policy_digest.clone(), + expires_at: Self::expires_at(self.config.policy_ttl_seconds), + ttl_less: false, + }; + self.write( + kv, + &self.policy_key, + entry, + self.config.policy_ttl_seconds, + work, + ) + .await; + if !work.expired() { + return Ok(policy_digest); + } + // The digest is not cached, so the next view could not find a + // plan skip written under it; skip at the policy level instead. + Reason::Timeout + } + Err(reason) => reason, + }; + let ttl = self.policy_skip_ttl(); + let entry = PolicyEntry::Skip { + reason, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write( + kv, + &self.policy_key, + entry, + ttl, + &mut Self::skip_deadline(services), + ) + .await; + Err(reason) + } + + fn policy_skip_ttl(&self) -> u64 { + self.config + .policy_ttl_seconds + .min(self.config.negative_ttl_seconds) + } + + async fn request_policy( + &self, + request: &PageRequest, + services: &RuntimeServices, + work: &mut Deadline<'_>, + ) -> Result { + let mut url = url::Url::parse(&self.policy_url).map_err(|_| Reason::RequestBuild)?; + url.query_pairs_mut() + .append_pair("page_url", &request.page_url); + let body = self + .fetch( + services, + Method::GET, + url.as_str(), + EdgeBody::empty(), + api::MAX_POLICY_RESPONSE_BYTES, + work, + ) + .await?; + api::parse_policy_response(&body).map_err(|error| parse_error_reason(&error)) + } + + async fn evict_policy(&self, kv: &dyn PlatformKvStore, deadline: &mut Deadline<'_>) { + if matches!( + deadline.race(kv.delete(&self.policy_key)).await, + Some(Ok(())) + ) { + return; + } + // If deletion fails, try to replace the stale digest with a bounded skip. + let ttl = self.policy_skip_ttl(); + let entry = PolicyEntry::Skip { + reason: Reason::PolicyChanged, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write(kv, &self.policy_key, entry, ttl, deadline).await; + log::debug!("encypher: policy eviction failed"); + } + + /// Call the sign API once. Every failure maps to a reason code. + async fn request_plan( + &self, + request: &PageRequest, + services: &RuntimeServices, + html: &str, + article: &Article, + policy_digest: &str, + work: &mut Deadline<'_>, + ) -> Result<(EmbeddingPlan, Option), Reason> { + let title = text::extract_title(html); + let body = serde_json::to_vec(&SignRequest { + text: &article.text, + page_url: &request.page_url, + document_title: title.as_deref(), + boundary_selector: &article.boundary.selector, + expected_policy_digest: policy_digest, + }) + .map_err(|_| Reason::RequestBuild)?; + let body = self + .fetch( + services, + Method::POST, + &self.sign_url, + EdgeBody::from(body), + api::MAX_RESPONSE_BYTES, + work, + ) + .await?; + let parsed = api::parse_sign_response(&body).map_err(|error| parse_error_reason(&error))?; + if !parsed.success { + return Err(if parsed.error.as_deref() == Some("quota_exceeded") { + Reason::QuotaExceeded + } else { + Reason::SignRejected + }); + } + if parsed.policy_digest.as_deref() != Some(policy_digest) { + return Err(Reason::PolicyDigestMismatch); + } + let plan = parsed.embedding_plan.ok_or(Reason::NoPlan)?; + Ok((plan, parsed.manifest_url)) + } + + /// One bounded HTTP call; never retried. + async fn fetch( + &self, + services: &RuntimeServices, + method: Method, + url: &str, + body: EdgeBody, + max_response_bytes: usize, + work: &mut Deadline<'_>, + ) -> Result { + let mut token = HeaderValue::from_str(self.config.worker_token.expose().trim()) + .map_err(|_| Reason::RequestBuild)?; + token.set_sensitive(true); + let http_request = Request::builder() + .method(method) + .uri(url) + .header(header::CONTENT_TYPE, "application/json") + .header(WORKER_TOKEN_HEADER, token) + .body(body) + .map_err(|_| Reason::RequestBuild)?; + let backend = ensure_integration_backend_with_timeout( + services, + &self.config.api_base_url, + INTEGRATION_ID, + Duration::from_millis(self.config.timeout_ms), + ) + .map_err(|_| Reason::Backend)?; + + let response = match work + .race( + services + .http_client() + .send(PlatformHttpRequest::new(http_request, backend)), + ) + .await + { + None => return Err(Reason::Timeout), + Some(Err(_)) => return Err(Reason::HttpError), + Some(Ok(response)) => response.response, + }; + let status = response.status(); + if status != StatusCode::OK { + return Err(match status.as_u16() { + 403 => Reason::Forbidden, + 409 => Reason::PolicyChanged, + 422 => Reason::Unprocessable, + 429 => Reason::RateLimited, + 500..=599 => Reason::ServerError, + _ => Reason::UnexpectedStatus, + }); + } + // One byte over the limit is enough for the parser to report TooLarge. + match work + .race( + response + .into_body() + .into_bytes_bounded(max_response_bytes.saturating_add(1)), + ) + .await + { + None => Err(Reason::Timeout), + Some(Err(_)) => Err(Reason::ResponseTooLarge), + Some(Ok(body)) => Ok(body), + } + } + + /// Read a fresh entry. Expired entries are misses; in TTL-less write mode + /// they are also deleted, best effort, so the store cleans itself. + async fn read( + &self, + kv: &dyn PlatformKvStore, + key: &str, + deadline: &mut Deadline<'_>, + ) -> Result, Reason> { + let bytes = match deadline.race(kv.get_bytes(key)).await { + None => return Err(Reason::Timeout), + Some(Err(KvError::Unavailable)) => return Err(Reason::KvUnavailable), + Some(Err(_)) => return Err(Reason::KvError), + Some(Ok(None)) => return Ok(None), + Some(Ok(Some(bytes))) => bytes, + }; + let entry: T = serde_json::from_slice(&bytes).map_err(|_| Reason::CachedEntryInvalid)?; + // An entry written without a TTL shows the mode to an instance that has + // not written yet, as on Spin, which instantiates the module per request. + if entry.ttl_less() { + self.ttl_less.store(true, Ordering::Relaxed); + } + if is_fresh(entry.expires_at(), unix_now()) { + return Ok(Some(entry)); + } + if self.ttl_less.load(Ordering::Relaxed) + && !matches!(deadline.race(kv.delete(key)).await, Some(Ok(()))) + { + log::debug!("encypher: expired cache entry could not be deleted"); + } + Ok(None) + } + + /// Best-effort cache write; failures are logged and ignored. + /// + /// Writes carry a TTL until the store first reports TTL writes as + /// unsupported; from then on this instance writes without one, and marks + /// the entry so a later instance that reads it knows the mode too. + async fn write( + &self, + kv: &dyn PlatformKvStore, + key: &str, + mut entry: E, + ttl_seconds: u64, + deadline: &mut Deadline<'_>, + ) { + if !self.ttl_less.load(Ordering::Relaxed) { + let Some(value) = encode(&entry) else { + return; + }; + let ttl = Duration::from_secs(ttl_seconds); + match deadline.race(kv.put_bytes_with_ttl(key, value, ttl)).await { + Some(Ok(())) => return, + Some(Err(KvError::Unsupported { .. })) => { + self.ttl_less.store(true, Ordering::Relaxed); + log::info!( + "encypher: the KV store does not support TTL writes; entries expire by their expires_at" + ); + } + Some(Err(_)) => { + log::debug!("encypher: cache write failed"); + return; + } + None => { + log::debug!("encypher: cache write timed out"); + return; + } + } + } + entry.mark_ttl_less(); + let Some(value) = encode(&entry) else { + return; + }; + match deadline.race(kv.put_bytes(key, value)).await { + Some(Ok(())) => {} + Some(Err(_)) => log::debug!("encypher: cache write failed"), + None => log::debug!("encypher: cache write timed out"), + } + } + + /// Write a negative entry unless a fresh, valid plan is already cached for the key. + /// + /// The KV trait has no insert-if-absent, so this re-reads first: a concurrent + /// successful sign for the same text keeps its plan. + async fn write_skip( + &self, + kv: &dyn PlatformKvStore, + key: &str, + reason: Reason, + scalars: usize, + deadline: &mut Deadline<'_>, + ) { + if let Ok(Some(CacheEntry::Plan { + plan, manifest_url, .. + })) = self.read::(kv, key, deadline).await + && api::validate_plan(&plan, manifest_url.as_deref(), scalars).is_ok() + { + return; + } + let ttl = self.config.negative_ttl_seconds; + let entry = CacheEntry::Skip { + reason, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write(kv, key, entry, ttl, deadline).await; + } +} + +/// Serialize a cache entry; a failure is logged and the write skipped. +fn encode(entry: &impl Serialize) -> Option { + match serde_json::to_vec(entry) { + Ok(value) => Some(Bytes::from(value)), + Err(_) => { + log::debug!("encypher: cache entry could not be serialized"); + None + } + } +} + +/// Apply a plan that passed [`api::validate_plan`] against the signed text, and +/// add the manifest meta tag when a manifest URL was returned. +fn render( + html: &str, + boundary: &Boundary, + plan: &EmbeddingPlan, + manifest_url: Option<&str>, +) -> Option { + let signed = embed::apply_plan(html, boundary, plan)?; + Some(match manifest_url { + Some(url) => embed::inject_manifest_meta(&signed, url), + None => signed, + }) +} + +fn parse_error_reason(error: &ApiError) -> Reason { + match error { + ApiError::TooLarge => Reason::ResponseTooLarge, + ApiError::Malformed => Reason::MalformedResponse, + } +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use std::collections::HashMap; + use std::sync::atomic::AtomicUsize; + use std::sync::{LazyLock, Mutex}; + use std::task::Poll; + + use error_stack::Report; + use serde_json::json; + + use super::*; + use trusted_server_core::platform::test_support::{ + StoppedTimer, StubHttpClient, build_services_with_http_client, + }; + use trusted_server_core::platform::{ + PlatformError, PlatformHttpClient, PlatformPendingRequest, PlatformResponse, + PlatformSelectResult, + }; + + const TOKEN: &str = "cwk_test_worker_token_value"; + const PAGE_URL: &str = "https://www.publisher.example.com/news/harbor-vote"; + const MARKER: &str = "\u{E0100}\u{E0101}\u{FE01}"; + const MANIFEST_URL: &str = "https://api.example.com/manifests/doc-1"; + const POLICY_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const ARTICLE: &str = "Harbor Council Votes\ +

Harbor council votes on the ferry schedule

\ +

The harbor council met on Tuesday evening and approved a revised ferry schedule \ + for the coming season.

Residents asked for earlier departures on weekdays.

\ +
Footer
"; + /// Simulated time that passes each time a [`TickTimer`] sleep is polled. + const TICK: Duration = Duration::from_millis(100); + + // --- test doubles ------------------------------------------------------- + + /// Return `Pending` `times` times, waking the task each time, then `Ready`. + async fn yield_pending(times: usize) { + for _ in 0..times { + let mut yielded = false; + futures::future::poll_fn(|cx| { + if yielded { + Poll::Ready(()) + } else { + yielded = true; + cx.waker().wake_by_ref(); + Poll::Pending + } + }) + .await; + } + } + + /// A timer on simulated time: each poll of a sleep advances it by [`TICK`]. + /// Sleeps only advance while the work they race is pending, as on a real + /// runtime where the clock runs while I/O is outstanding. + #[derive(Default)] + struct TickTimer { + requested: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl PlatformTimer for TickTimer { + async fn sleep(&self, duration: Duration) { + self.requested + .lock() + .expect("should lock requested durations") + .push(duration); + let ticks = duration.as_millis().div_ceil(TICK.as_millis()); + yield_pending(usize::try_from(ticks).expect("ticks should fit")).await; + } + } + + /// In-memory KV store that counts calls and can fail, stall or refuse TTLs. + #[derive(Default)] + struct MemoryKv { + entries: Mutex>, + ttls: Mutex>, + deletes: Mutex>, + reads: AtomicUsize, + ttl_writes: AtomicUsize, + /// Plan-key reads that report a miss before reads see the stored entry. + hidden_reads: AtomicUsize, + /// Every read fails with [`KvError::Unavailable`], as an unbound store does. + unavailable: bool, + /// Every read fails with an error other than unavailability. + fail_reads: bool, + /// Every read never completes. + hung_reads: bool, + /// Writes to keys with this prefix never complete. + hung_writes_to: Option<&'static str>, + /// The first write to this exact key never completes; later writes succeed. + hung_first_write_to: Option, + hung_first_write_taken: AtomicBool, + /// TTL writes fail with [`KvError::Unsupported`], as on Spin. + ttl_unsupported: bool, + /// `Pending` polls before every operation completes. + pending_polls: usize, + } + + impl MemoryKv { + fn entries(&self) -> HashMap { + self.entries.lock().expect("should lock entries").clone() + } + + fn entry(&self, key: &str) -> serde_json::Value { + serde_json::from_slice(&self.entries()[key]).expect("should store JSON") + } + + fn insert(&self, key: &str, value: &serde_json::Value) { + self.entries.lock().expect("should lock entries").insert( + key.to_owned(), + Bytes::from(serde_json::to_vec(value).expect("should serialize")), + ); + } + + fn remove(&self, key: &str) { + self.entries + .lock() + .expect("should lock entries") + .remove(key); + } + + /// The single article entry under the current plan namespace. + fn only_entry(&self) -> (String, serde_json::Value) { + let entries: Vec<_> = self + .entries() + .into_keys() + .filter(|key| key.starts_with("encypher:v3:")) + .collect(); + assert_eq!(entries.len(), 1, "should hold exactly one article entry"); + let key = entries.into_iter().next().expect("should have an entry"); + let value = self.entry(&key); + (key, value) + } + + fn reads(&self) -> usize { + self.reads.load(Ordering::SeqCst) + } + + async fn store(&self, key: &str, value: Bytes) -> Result<(), KvError> { + yield_pending(self.pending_polls).await; + if self + .hung_writes_to + .is_some_and(|prefix| key.starts_with(prefix)) + || (self.hung_first_write_to.as_deref() == Some(key) + && !self.hung_first_write_taken.swap(true, Ordering::SeqCst)) + { + futures::future::pending::<()>().await; + } + self.entries + .lock() + .expect("should lock entries") + .insert(key.to_owned(), value); + Ok(()) + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + self.reads.fetch_add(1, Ordering::SeqCst); + yield_pending(self.pending_polls).await; + if self.hung_reads { + futures::future::pending::<()>().await; + } + if self.unavailable { + return Err(KvError::Unavailable); + } + if self.fail_reads { + return Err(KvError::LimitExceeded { + message: "test failure".to_owned(), + }); + } + if key.starts_with(CACHE_KEY_PREFIX) + && self + .hidden_reads + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + return Ok(None); + } + Ok(self + .entries + .lock() + .expect("should lock entries") + .get(key) + .cloned()) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.store(key, value).await + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + self.ttl_writes.fetch_add(1, Ordering::SeqCst); + if self.ttl_unsupported { + return Err(KvError::Unsupported { + operation: "put_bytes_with_ttl".to_owned(), + }); + } + self.ttls + .lock() + .expect("should lock TTLs") + .insert(key.to_owned(), ttl); + self.store(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + yield_pending(self.pending_polls).await; + self.deletes + .lock() + .expect("should lock deletes") + .push(key.to_owned()); + self.remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(edgezero_core::key_value_store::KvPage::default()) + } + } + + /// HTTP client whose send never completes. + #[derive(Default)] + struct HungHttpClient { + sends: AtomicUsize, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for HungHttpClient { + async fn send( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + self.sends.fetch_add(1, Ordering::SeqCst); + futures::future::pending().await + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + } + + /// Wraps [`StubHttpClient`] so the send and the response body each return + /// `Pending` before they are ready, as on Cloudflare, Spin and Axum. + struct YieldingHttpClient { + inner: Arc, + pending_polls: usize, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for YieldingHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + yield_pending(self.pending_polls).await; + let response = self.inner.send(request).await?; + let (parts, body) = response.response.into_parts(); + let bytes = body + .into_bytes_bounded(usize::MAX) + .await + .expect("should read the stub body"); + let polls = self.pending_polls; + let body = EdgeBody::stream(futures::stream::once(async move { + yield_pending(polls).await; + bytes + })); + Ok(PlatformResponse::new(http::Response::from_parts( + parts, body, + ))) + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + } + + /// Records log lines, with their level, emitted on the current thread. + struct ThreadLogger; + + thread_local! { + static LOGS: RefCell> = const { RefCell::new(Vec::new()) }; + } + + impl log::Log for ThreadLogger { + fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool { + true + } + + fn log(&self, record: &log::Record<'_>) { + LOGS.with(|logs| { + logs.borrow_mut() + .push(format!("{} {}", record.level(), record.args())); + }); + } + + fn flush(&self) {} + } + + static LOGGER: ThreadLogger = ThreadLogger; + + /// A sign-failure case: name and the API response, `None` for a transport error. + type FailureCase = (&'static str, Option<(u16, Vec)>); + /// A gate case: name, signer, and the request mutation that trips the gate. + type GateCase<'a> = (&'static str, &'a Arc, fn(&mut PageRequest)); + + /// Runs `run` with this thread's log lines captured. Returns `None`, after + /// skipping `run`, when another logger already owns the process. + fn capture_logs(run: impl FnOnce()) -> Option> { + static INSTALLED: LazyLock = LazyLock::new(|| { + let installed = log::set_logger(&LOGGER).is_ok(); + if installed { + log::set_max_level(log::LevelFilter::Trace); + } + installed + }); + if !*INSTALLED { + return None; + } + LOGS.with(|logs| logs.borrow_mut().clear()); + run(); + Some(LOGS.with(RefCell::take)) + } + + // --- fixtures ----------------------------------------------------------- + + fn config_json(token: &str) -> serde_json::Value { + json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": token, + }) + } + + fn factory_with(config: serde_json::Value, max_document_bytes: usize) -> Arc { + let config: EncypherConfig = serde_json::from_value(config).expect("should parse config"); + Arc::new(Signer::new(config, max_document_bytes)) + } + + fn factory() -> Arc { + factory_with(config_json(TOKEN), 5_000_000) + } + + fn policy_key() -> String { + factory().policy_key.clone() + } + + /// Services without a timer, as on Fastly. + fn services(http: Arc, kv: &Arc) -> RuntimeServices { + build_services_with_http_client(http) + .with_kv_store(Arc::clone(kv) as Arc) + } + + /// Services with a [`TickTimer`], as on Cloudflare, Spin and Axum. + fn timed_services(http: Arc, kv: &Arc) -> RuntimeServices { + services(http, kv).with_timer(Arc::new(TickTimer::default())) + } + + fn request() -> PageRequest { + PageRequest { + method: Method::GET, + page_url: PAGE_URL.to_owned(), + response_status: 200, + origin_response_is_private: false, + } + } + + fn plan_json(marker: &str) -> serde_json::Value { + json!({ + "index_unit": "codepoint", + "operations": [{ "insert_after_index": 5, "marker": marker }], + }) + } + + fn sign_response(marker: &str) -> Vec { + sign_response_for_policy(marker, POLICY_DIGEST) + } + + fn policy_response(digest: &str) -> Vec { + serde_json::to_vec(&json!({"policy_digest": digest})).expect("should serialize policy") + } + + fn sign_response_for_policy(marker: &str, digest: &str) -> Vec { + serde_json::to_vec(&json!({ + "success": true, + "policy_digest": digest, + "embedding_plan": plan_json(marker), + "manifest_url": MANIFEST_URL, + "verification_url": "https://api.example.com/verify/doc-1", + "document_id": "doc-1", + })) + .expect("should serialize sign response") + } + + fn signing_http() -> Arc { + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + http.push_response(200, sign_response(MARKER)); + http + } + + fn seed_plan(kv: &Arc) -> String { + view(&factory(), &services(signing_http(), kv)) + } + + /// Drive one document the way the middleware and the publisher driver do: + /// the gates on the request, when there is one, then two chunks, then the + /// asynchronous finish with the core backstop. A request the gates refuse, + /// or none, holds nothing, so the document passes through. + fn run( + factory: &Arc, + request: Option, + services: &RuntimeServices, + html: &str, + ) -> String { + let Some(request) = request.and_then(|request| factory.admit(request).ok()) else { + return html.to_owned(); + }; + futures::executor::block_on(async { + let mut processor = document_processor(factory, request); + let (first, second) = html.as_bytes().split_at(html.len() / 2); + let mut output = processor + .process_chunk(first, false) + .expect("should process first chunk"); + output.extend( + processor + .finish(second, services) + .await + .expect("should finish the document"), + ); + String::from_utf8(output).expect("should emit UTF-8") + }) + } + + /// One view of [`ARTICLE`] at [`PAGE_URL`]. + fn view(factory: &Arc, services: &RuntimeServices) -> String { + run(factory, Some(request()), services, ARTICLE) + } + + fn assert_signed(output: &str) { + assert!(output.contains(MARKER), "should embed the plan marker"); + assert!( + output.contains("c2pa-manifest-url") && output.contains(MANIFEST_URL), + "should inject the manifest meta tag" + ); + assert_eq!( + output.matches(MARKER).count(), + 1, + "should add the marker once" + ); + } + + /// Rewrite the `expires_at` of the entry at `key` to `expires_at`. + fn set_expiry(kv: &MemoryKv, key: &str, expires_at: u64) { + let mut entry = kv.entry(key); + entry["expires_at"] = json!(expires_at); + kv.insert(key, &entry); + } + + fn expired() -> u64 { + unix_now().saturating_sub(1) + } + + // --- policy ------------------------------------------------------------- + + #[test] + fn policy_failures_are_negatively_cached_without_using_an_old_plan() { + let cases: Vec = vec![ + ("unavailable", Some((503, Vec::new()))), + ("forbidden", Some((403, Vec::new()))), + ("transport", None), + ("malformed", Some((200, b"not json".to_vec()))), + ("missing digest", Some((200, b"{}".to_vec()))), + ( + "uppercase", + Some((200, policy_response(&POLICY_DIGEST.to_uppercase()))), + ), + ( + "oversized", + Some((200, vec![b' '; api::MAX_POLICY_RESPONSE_BYTES + 2])), + ), + ]; + for (case, response) in cases { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + kv.remove(&policy_key()); + let http = Arc::new(StubHttpClient::new()); + if let Some((status, body)) = response { + http.push_response(status, body); + } + let services = services(Arc::clone(&http) as Arc, &kv); + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE, "{case}"); + } + assert_eq!(http.recorded_request_methods(), ["GET"], "{case}"); + assert_eq!(kv.entry(&policy_key())["kind"], "skip", "{case}"); + assert_eq!( + kv.ttls.lock().expect("lock")[&policy_key()], + Duration::from_secs(300) + ); + // Once the skip is gone, refresh may reuse only the matching old plan. + kv.remove(&policy_key()); + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["GET", "GET"]); + } + } + + #[test] + fn malformed_cached_policy_cannot_unlock_a_cached_article() { + let future = unix_now() + 300; + for value in [ + json!("not an entry"), + json!({"kind": "digest", "policy_digest": null, "expires_at": future}), + json!({"kind": "digest", "policy_digest": "bad", "expires_at": future}), + json!({ + "kind": "digest", "policy_digest": POLICY_DIGEST, "expires_at": future, + "extra": true, + }), + // `expires_at` is required. + json!({"kind": "digest", "policy_digest": POLICY_DIGEST}), + ] { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + kv.insert(&policy_key(), &value); + let http = Arc::new(StubHttpClient::new()); + let services = services(Arc::clone(&http) as Arc, &kv); + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE, "{value}"); + } + assert!(http.recorded_request_methods().is_empty(), "{value}"); + } + } + + #[test] + fn policy_cache_hit_and_article_or_policy_changes_select_the_correct_plan() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, sign_response(MARKER)); + let services = services(Arc::clone(&http) as Arc, &kv); + let changed = ARTICLE.replace("Tuesday", "Wednesday"); + assert_signed(&run(&factory(), Some(request()), &services, &changed)); + assert_eq!(http.recorded_request_methods(), ["POST"]); + + kv.remove(&policy_key()); + let next_digest = "b".repeat(64); + http.push_response(200, policy_response(&next_digest)); + http.push_response(200, sign_response_for_policy(MARKER, &next_digest)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["POST", "GET", "POST"]); + let count = http.recorded_request_methods().len(); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods().len(), count); + // Returning to the original policy selects its original plan, not the intervening one. + kv.remove(&policy_key()); + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_signed(&view(&factory(), &services)); + assert_eq!( + http.recorded_request_methods(), + ["POST", "GET", "POST", "GET"] + ); + } + + #[test] + fn conflict_or_response_digest_mismatch_evicts_policy_without_retrying() { + let mut missing: serde_json::Value = + serde_json::from_slice(&sign_response(MARKER)).expect("JSON"); + missing + .as_object_mut() + .expect("object") + .remove("policy_digest"); + for (status, body) in [ + ( + 409, + br#"{"detail":"TRUSTED_SERVER_POLICY_CHANGED"}"#.to_vec(), + ), + (409, b"unparseable conflict".to_vec()), + (200, sign_response_for_policy(MARKER, &"b".repeat(64))), + (200, sign_response_for_policy(MARKER, "invalid")), + (200, serde_json::to_vec(&missing).expect("JSON")), + ] { + let kv = Arc::new(MemoryKv::default()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + http.push_response(status, body); + let services = services(Arc::clone(&http) as Arc, &kv); + assert_eq!(view(&factory(), &services), ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert!(!kv.entries().contains_key(&policy_key())); + assert_eq!(kv.only_entry().1["kind"], "skip"); + // Next request refetches policy, but the failed article remains negatively cached. + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_eq!(view(&factory(), &services), ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET", "POST", "GET"]); + } + } + + #[test] + fn entries_expire_at_their_ttl_and_the_store_ttl_matches() { + let kv = Arc::new(MemoryKv::default()); + let before = unix_now(); + seed_plan(&kv); + let after = unix_now(); + let ttls = kv.ttls.lock().expect("lock").clone(); + let (plan_key, plan) = kv.only_entry(); + for (key, entry, ttl) in [ + (policy_key(), kv.entry(&policy_key()), 300), + (plan_key, plan, 604_800), + ] { + assert_eq!(ttls[&key], Duration::from_secs(ttl)); + let expires_at = entry["expires_at"] + .as_u64() + .expect("should carry expires_at"); + assert!( + (before + ttl..=after + ttl).contains(&expires_at), + "{key} should expire {ttl}s after it was written" + ); + } + for (policy_ttl, negative_ttl, expected) in [(60, 300, 60), (300, 60, 60)] { + let kv = Arc::new(MemoryKv::default()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + let mut config = config_json(TOKEN); + config["policy_ttl_seconds"] = json!(policy_ttl); + config["negative_ttl_seconds"] = json!(negative_ttl); + let factory = factory_with(config, 5_000_000); + assert_eq!(view(&factory, &services(http, &kv)), ARTICLE); + assert_eq!( + kv.ttls.lock().expect("lock")[&factory.policy_key], + Duration::from_secs(expected) + ); + } + } + + // --- sign path and cache ------------------------------------------------ + + #[test] + fn signs_on_a_miss_with_the_worker_token_and_caches_the_plan() { + let http = signing_http(); + let kv = Arc::new(MemoryKv::default()); + let services = services(Arc::clone(&http) as Arc, &kv); + + let output = view(&factory(), &services); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert_eq!( + http.recorded_request_uris(), + [ + format!( + "https://api.example.com/api/v1/public/cdn/trusted-server/policy?page_url={}", + url::form_urlencoded::byte_serialize(PAGE_URL.as_bytes()).collect::() + ), + "https://api.example.com/api/v1/public/cdn/sign".to_owned(), + ] + ); + let headers = &http.recorded_request_headers()[1]; + let header = |name: &str| { + headers + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + }; + assert_eq!(header("x-encypher-worker-token"), Some(TOKEN)); + assert_eq!(header("content-type"), Some("application/json")); + assert_eq!( + header("idempotency-key"), + None, + "should send no idempotency key" + ); + let body: serde_json::Value = serde_json::from_slice(&http.recorded_request_bodies()[1]) + .expect("should send a JSON body"); + assert_eq!(body["page_url"], PAGE_URL); + assert_eq!(body["expected_policy_digest"], POLICY_DIGEST); + assert_eq!(body["document_title"], "Harbor Council Votes"); + assert_eq!(body["boundary_selector"], "article"); + let sent_text = body["text"].as_str().expect("should send text"); + assert!( + sent_text.starts_with("Harbor council votes on the ferry schedule The harbor council"), + "should send the article's visible text, got {sent_text}" + ); + assert!(!sent_text.contains("Footer") && !sent_text.contains("Home")); + let (key, entry) = kv.only_entry(); + assert!( + key.starts_with("encypher:v3:") && key.contains(POLICY_DIGEST), + "unexpected cache key {key}" + ); + assert!(!key.contains(TOKEN), "cache key must not contain the token"); + assert!( + policy_key().starts_with("encypher:policy:v2:"), + "unexpected policy key" + ); + assert_eq!(entry["kind"], "plan"); + assert_eq!(entry["plan"], plan_json(MARKER)); + } + + #[test] + fn a_cached_plan_signs_without_calling_the_api() { + let kv = Arc::new(MemoryKv::default()); + let first = seed_plan(&kv); + + let http = Arc::new(StubHttpClient::new()); + let second = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!( + second, first, + "a cache hit should serve the same signed page" + ); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + + #[test] + fn a_cached_skip_passes_through_without_calling_the_api() { + let kv = Arc::new(MemoryKv::default()); + let first_http = Arc::new(StubHttpClient::new()); + first_http.push_response(200, policy_response(POLICY_DIGEST)); + first_http.push_response(403, b"{}".to_vec()); + view(&factory(), &services(first_http, &kv)); + assert_eq!(kv.only_entry().1["kind"], "skip"); + + let http = Arc::new(StubHttpClient::new()); + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + + #[test] + fn a_failure_does_not_replace_a_fresh_plan_written_since_the_first_read() { + let seeded = Arc::new(MemoryKv::default()); + seed_plan(&seeded); + // The first read misses, as if a concurrent request wrote the plan after it. + let kv = Arc::new(MemoryKv { + entries: Mutex::new(seeded.entries()), + hidden_reads: AtomicUsize::new(1), + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "the failed request should pass through"); + assert_eq!(http.recorded_backend_names().len(), 1); + assert_eq!(kv.only_entry().1["kind"], "plan", "the plan should survive"); + } + + #[test] + fn sign_failures_pass_through_and_write_a_skip() { + let oversized = { + let mut body = sign_response(MARKER); + body.resize(api::MAX_RESPONSE_BYTES + 2, b' '); + body + }; + let quota = serde_json::to_vec(&json!({ "success": false, "error": "quota_exceeded" })) + .expect("should serialize"); + let insecure_manifest = serde_json::to_vec(&json!({ + "success": true, + "policy_digest": POLICY_DIGEST, + "embedding_plan": plan_json(MARKER), + "manifest_url": "javascript:alert(1)", + })) + .expect("should serialize"); + let cases: Vec = vec![ + ("403", Some((403, b"{}".to_vec()))), + ("422", Some((422, b"{}".to_vec()))), + ("429", Some((429, b"{}".to_vec()))), + ("5xx", Some((502, Vec::new()))), + ("transport error or first-byte timeout", None), + ("oversized body", Some((200, oversized))), + ("malformed body", Some((200, b"not json".to_vec()))), + ( + "invalid marker", + Some((200, sign_response(""))), + ), + ("quota", Some((200, quota))), + ("non-https manifest url", Some((200, insecure_manifest))), + ( + "null plan", + Some(( + 200, + serde_json::to_vec(&json!({ + "success": true, "policy_digest": POLICY_DIGEST, "embedding_plan": null + })) + .expect("JSON"), + )), + ), + ]; + + for (case, response) in cases { + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + if let Some((status, body)) = response { + http.push_response(status, body); + } + let kv = Arc::new(MemoryKv::default()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!( + output, ARTICLE, + "{case}: should pass the page through unchanged" + ); + assert_eq!( + http.recorded_backend_names().len(), + 2, + "{case}: one policy call and one sign call" + ); + assert_eq!( + kv.only_entry().1["kind"], + "skip", + "{case}: should cache a skip" + ); + } + } + + #[test] + fn a_tampered_cached_plan_is_rejected_without_calling_the_api() { + for (field, value) in [ + ("/plan/operations/0/marker", ""), + ("/manifest_url", "http://manifest.example.com/doc-1"), + ] { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let (key, mut entry) = kv.only_entry(); + *entry + .pointer_mut(field) + .expect("seeded entry should have the field") = json!(value); + kv.insert(&key, &entry); + + let http = Arc::new(StubHttpClient::new()); + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "{field}: should pass through"); + assert!( + http.recorded_backend_names().is_empty(), + "{field}: should make no API call" + ); + } + } + + #[test] + fn a_changed_credential_misses_the_cache() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + + let http = signing_http(); + let output = view( + &factory_with(config_json("cwk_rotated_token"), 5_000_000), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert_eq!( + kv.entries().len(), + 4, + "each credential should have its own policy and article entries" + ); + } + + // --- asynchronous resolve and the hold deadline ------------------------- + + #[test] + fn calls_that_are_pending_before_ready_are_awaited_and_sign() { + let timers: [Option>; 3] = [ + None, + Some(Arc::new(StoppedTimer)), + Some(Arc::new(TickTimer::default())), + ]; + for timer in timers { + let label = if timer.is_some() { "timer" } else { "no timer" }; + let kv = Arc::new(MemoryKv { + pending_polls: 2, + ..MemoryKv::default() + }); + let stub = signing_http(); + let http = Arc::new(YieldingHttpClient { + inner: Arc::clone(&stub), + pending_polls: 2, + }); + let mut services = services(http, &kv); + if let Some(timer) = timer { + services = services.with_timer(timer); + } + + let first = view(&factory(), &services); + let second = view(&factory(), &services); + + assert_signed(&first); + assert_eq!( + second, first, + "{label}: the second view should hit the cache" + ); + assert_eq!(stub.recorded_request_methods(), ["GET", "POST"], "{label}"); + assert_eq!(kv.only_entry().1["kind"], "plan", "{label}"); + } + } + + #[test] + fn a_hung_policy_call_times_out_writes_a_skip_and_the_next_view_makes_no_call() { + let kv = Arc::new(MemoryKv::default()); + let hung = Arc::new(HungHttpClient::default()); + + let output = view( + &factory(), + &timed_services(Arc::clone(&hung) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "should release the page unchanged"); + assert_eq!(hung.sends.load(Ordering::SeqCst), 1); + let skip = kv.entry(&policy_key()); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + + let http = Arc::new(StubHttpClient::new()); + let next = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + assert_eq!(next, ARTICLE); + assert!( + http.recorded_backend_names().is_empty(), + "the next view should be a cached skip" + ); + } + + #[test] + fn a_hung_sign_call_or_body_times_out_writes_a_plan_skip_and_the_next_view_makes_no_call() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let article_key = kv.only_entry().0; + kv.remove(&article_key); + let hung = Arc::new(HungHttpClient::default()); + let services = timed_services(Arc::clone(&hung) as Arc, &kv); + + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE); + } + + assert_eq!( + hung.sends.load(Ordering::SeqCst), + 1, + "one sign call in total" + ); + let (_, skip) = kv.only_entry(); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + } + + #[test] + fn a_hung_kv_read_times_out_without_calling_the_api() { + let kv = Arc::new(MemoryKv { + hung_reads: true, + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 1); + assert!(http.recorded_backend_names().is_empty()); + } + + #[test] + fn a_plan_write_that_runs_out_of_time_still_returns_the_signed_page() { + let kv = Arc::new(MemoryKv { + hung_writes_to: Some(CACHE_KEY_PREFIX), + ..MemoryKv::default() + }); + let http = signing_http(); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert!( + kv.entries() + .keys() + .all(|key| !key.starts_with(CACHE_KEY_PREFIX)), + "the abandoned plan write should store nothing" + ); + } + + #[test] + fn a_policy_digest_write_that_runs_out_of_time_caches_a_policy_skip() { + let kv = Arc::new(MemoryKv { + hung_first_write_to: Some(policy_key()), + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "should release the page unchanged"); + assert_eq!( + http.recorded_request_methods(), + ["GET"], + "no sign call after the digest write ran out of time" + ); + let skip = kv.entry(&policy_key()); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + assert!( + kv.entries() + .keys() + .all(|key| !key.starts_with(CACHE_KEY_PREFIX)), + "no plan skip under a digest the next view cannot know" + ); + + // A hung client turns any API call into a full hold, so zero sends + // after one KV read shows the next view is a cached skip. + let reads = kv.reads(); + let hung = Arc::new(HungHttpClient::default()); + let next = view( + &factory(), + &timed_services(Arc::clone(&hung) as Arc, &kv), + ); + assert_eq!(next, ARTICLE); + assert_eq!(hung.sends.load(Ordering::SeqCst), 0, "no API call"); + assert_eq!(kv.reads() - reads, 1, "only the policy entry is read"); + } + + #[test] + fn the_core_backstop_lies_beyond_the_module_hold_bound() { + let kv = Arc::new(MemoryKv::default()); + let timer = Arc::new(TickTimer::default()); + let services = services(Arc::new(HungHttpClient::default()), &kv) + .with_timer(Arc::clone(&timer) as Arc); + + assert_eq!(view(&factory(), &services), ARTICLE); + + let max_hold = Duration::from_millis(5_000); + let requested = timer.requested.lock().expect("lock").clone(); + let (backstop, module): (Vec<_>, Vec<_>) = + requested.into_iter().partition(|sleep| *sleep > max_hold); + assert_eq!(backstop.len(), 1, "one core backstop beyond max_hold_ms"); + // The skip-write reserve starts only after the work deadline, so the + // module's own sleeps run back to back. + assert!( + !module.is_empty() && module.iter().sum::() <= max_hold, + "module sleeps {module:?} should fit within max_hold_ms" + ); + assert_eq!( + kv.entry(&policy_key())["reason"], + "timeout", + "the module deadline, not the backstop, should release the page" + ); + } + + // --- expiry, write mode and key namespaces ------------------------------ + + #[test] + fn expired_entries_are_misses() { + // (entry to expire, API calls the next view makes) + let cases: [(&str, &[&str]); 4] = [ + ("policy digest", &["GET"]), + ("policy skip", &["GET"]), + ("plan", &["POST"]), + ("plan skip", &["POST"]), + ]; + for (case, expected_calls) in cases { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let (plan_key, _) = kv.only_entry(); + let key = if case.starts_with("policy") { + policy_key() + } else { + plan_key + }; + if case.ends_with("skip") { + kv.insert( + &key, + &json!({"kind": "skip", "reason": "server_error", "expires_at": expired()}), + ); + } else { + set_expiry(&kv, &key, expired()); + } + let http = Arc::new(StubHttpClient::new()); + if case.starts_with("policy") { + http.push_response(200, policy_response(POLICY_DIGEST)); + } else { + http.push_response(200, sign_response(MARKER)); + } + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), expected_calls, "{case}"); + assert!( + kv.entry(&key)["expires_at"].as_u64() > Some(unix_now()), + "{case}: should be replaced by a fresh entry" + ); + } + } + + #[test] + fn an_expired_plan_does_not_block_a_skip_write() { + let seeded = Arc::new(MemoryKv::default()); + seed_plan(&seeded); + let kv = Arc::new(MemoryKv { + entries: Mutex::new(seeded.entries()), + hidden_reads: AtomicUsize::new(1), + ..MemoryKv::default() + }); + let (plan_key, _) = kv.only_entry(); + set_expiry(&kv, &plan_key, expired()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + + assert_eq!(view(&factory(), &services(http, &kv)), ARTICLE); + + assert_eq!(kv.entry(&plan_key)["kind"], "skip"); + } + + #[test] + fn a_store_without_ttl_writes_switches_once_and_expires_by_value() { + let kv = Arc::new(MemoryKv { + ttl_unsupported: true, + ..MemoryKv::default() + }); + let factory = factory(); + let http = signing_http(); + let services = services(Arc::clone(&http) as Arc, &kv); + + let first = view(&factory, &services); + + assert_signed(&first); + assert_eq!( + kv.ttl_writes.load(Ordering::SeqCst), + 1, + "only the first write should try a TTL" + ); + let (plan_key, plan) = kv.only_entry(); + assert_eq!(plan["kind"], "plan"); + assert!(plan["expires_at"].as_u64() > Some(unix_now())); + assert_eq!(kv.entry(&policy_key())["kind"], "digest"); + assert_eq!(view(&factory, &services), first, "should hit the cache"); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + + // Expired entries are misses and are deleted when read. + set_expiry(&kv, &plan_key, expired()); + http.push_response(200, sign_response(MARKER)); + assert_signed(&view(&factory, &services)); + assert_eq!(*kv.deletes.lock().expect("lock"), [plan_key.as_str()]); + assert_eq!( + kv.entry(&plan_key)["kind"], + "plan", + "should store a new plan" + ); + } + + #[test] + fn a_fresh_instance_deletes_expired_entries_written_without_a_ttl() { + // Spin instantiates the module per request, so the instance that reads + // an expired entry has not written anything yet. + let kv = Arc::new(MemoryKv { + ttl_unsupported: true, + ..MemoryKv::default() + }); + view(&factory(), &services(signing_http(), &kv)); + set_expiry(&kv, &policy_key(), expired()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET"]); + assert_eq!(*kv.deletes.lock().expect("lock"), [policy_key()]); + } + + #[test] + fn expired_entries_are_not_deleted_on_a_store_with_ttls() { + let kv = Arc::new(MemoryKv::default()); + let http = signing_http(); + let services = services(Arc::clone(&http) as Arc, &kv); + view(&factory(), &services); + let (plan_key, _) = kv.only_entry(); + set_expiry(&kv, &plan_key, expired()); + http.push_response(200, sign_response(MARKER)); + + assert_signed(&view(&factory(), &services)); + + assert!(kv.deletes.lock().expect("lock").is_empty()); + } + + #[test] + fn entries_under_the_previous_key_versions_are_never_read() { + let current = Arc::new(MemoryKv::default()); + seed_plan(¤t); + // The same plan and policy, as the previous release stored them. + let kv = Arc::new(MemoryKv::default()); + for (key, value) in current.entries() { + let mut entry: serde_json::Value = + serde_json::from_slice(&value).expect("should store JSON"); + entry.as_object_mut().expect("object").remove("expires_at"); + let old_key = key + .replace("encypher:v3:", "encypher:v2:") + .replace("encypher:policy:v2:", "encypher:policy:v1:"); + kv.insert(&old_key, &entry); + } + let old_entries = kv.entries(); + let http = signing_http(); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + let entries = kv.entries(); + assert_eq!(entries.len(), old_entries.len() * 2); + for (key, value) in old_entries { + assert_eq!(entries[&key], value, "{key} should be left untouched"); + } + } + + // --- KV failures -------------------------------------------------------- + + #[test] + fn kv_read_errors_pass_through_without_calling_the_api() { + for kv in [ + MemoryKv { + fail_reads: true, + ..MemoryKv::default() + }, + MemoryKv { + unavailable: true, + ..MemoryKv::default() + }, + ] { + let kv = Arc::new(kv); + let http = Arc::new(StubHttpClient::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 1); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + } + + #[test] + fn an_unbound_kv_store_logs_kv_unavailable_and_warns_once() { + let Some(logs) = capture_logs(|| { + let http: Arc = Arc::new(StubHttpClient::new()); + let unbound = Arc::new(MemoryKv { + unavailable: true, + ..MemoryKv::default() + }); + for _ in 0..2 { + view(&factory(), &services(Arc::clone(&http), &unbound)); + } + let failing = Arc::new(MemoryKv { + fail_reads: true, + ..MemoryKv::default() + }); + view(&factory(), &services(http, &failing)); + }) else { + // Log lines go to a logger installed elsewhere and cannot be observed here. + return; + }; + + let outcomes: Vec<_> = logs + .iter() + .filter(|line| line.contains("encypher: outcome=")) + .collect(); + assert_eq!( + outcomes, + [ + "INFO encypher: outcome=passed_through reason=kv_unavailable", + "INFO encypher: outcome=passed_through reason=kv_unavailable", + "INFO encypher: outcome=passed_through reason=kv_error", + ] + ); + assert_eq!( + logs.iter() + .filter(|line| line.starts_with("WARN") && line.contains("KV store is unavailable")) + .count(), + 1, + "should warn once per instance: {logs:?}" + ); + } + + // --- gates and skips ---------------------------------------------------- + + #[test] + fn gated_documents_pass_through_with_no_kv_or_api_io() { + let signer = factory(); + let cases: Vec<(GateCase<'_>, Gate)> = vec![ + (("method", &signer, |r| r.method = Method::POST), Gate::Method), + (("status", &signer, |r| r.response_status = 404), Gate::Status), + ( + ("origin private", &signer, |r| { + r.origin_response_is_private = true; + }), + Gate::OriginPrivate, + ), + ( + ("host mismatch", &signer, |r| { + r.page_url = "https://other.example.com/news/harbor-vote".to_owned(); + }), + Gate::HostMismatch, + ), + ( + ("excluded path", &signer, |r| { + r.page_url = "https://publisher.example.com/WP-Admin/post".to_owned(); + }), + Gate::ExcludedPath, + ), + ( + ("asset suffix", &signer, |r| { + r.page_url = "https://publisher.example.com/feed.XML".to_owned(); + }), + Gate::ExcludedPath, + ), + ( + ("invalid page url", &signer, |r| { + r.page_url = "not a url".to_owned(); + }), + Gate::InvalidPageUrl, + ), + ]; + + for ((case, signer, mutate), gate) in cases { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + let services = services(Arc::clone(&http) as Arc, &kv); + let mut request = request(); + mutate(&mut request); + + assert_eq!( + signer.admit(request.clone()), + Err(gate), + "{case}: should be refused by its gate" + ); + let output = run(signer, Some(request), &services, ARTICLE); + + assert_eq!(output, ARTICLE, "{case}: should pass through"); + assert_eq!(kv.reads(), 0, "{case}: should read no KV"); + assert!( + http.recorded_backend_names().is_empty(), + "{case}: no API call" + ); + } + + let kv = Arc::new(MemoryKv::default()); + assert_eq!( + run( + &factory(), + None, + &services(Arc::new(StubHttpClient::new()), &kv), + ARTICLE + ), + ARTICLE, + "nothing prepared" + ); + } + + #[test] + fn a_bare_domain_page_is_signed() { + let kv = Arc::new(MemoryKv::default()); + let mut request = request(); + request.page_url = "https://publisher.example.com/news/harbor-vote".to_owned(); + + let output = run( + &factory(), + Some(request), + &services(signing_http(), &kv), + ARTICLE, + ); + + assert_signed(&output); + } + + #[test] + fn an_oversized_document_is_flushed_and_passed_through() { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + + let output = view( + &factory_with(config_json(TOKEN), 64), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 0); + assert!(http.recorded_backend_names().is_empty()); + } + + #[test] + fn already_signed_or_short_pages_are_not_sent() { + let magic = "\u{E01DF}\u{E01E0}\u{E01E1}\u{E01E2}"; + let marked = ARTICLE.replace("The harbor council", &format!("The harbor{magic} council")); + let with_meta = ARTICLE.replace( + "", + r#""#, + ); + let short = "

Too short.

"; + + for page in [marked.as_str(), with_meta.as_str(), short] { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + + let output = run( + &factory(), + Some(request()), + &services(Arc::clone(&http) as Arc, &kv), + page, + ); + + assert_eq!(output, page); + assert_eq!(kv.reads(), 0, "should read no KV"); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + } + + #[test] + fn logs_carry_outcomes_but_never_the_token() { + let kv = Arc::new(MemoryKv::default()); + let Some(logs) = capture_logs(|| { + view(&factory(), &services(signing_http(), &kv)); + view(&factory(), &services(Arc::new(StubHttpClient::new()), &kv)); + let failing = Arc::new(StubHttpClient::new()); + failing.push_response(403, b"{}".to_vec()); + view( + &factory(), + &services(failing, &Arc::new(MemoryKv::default())), + ); + }) else { + // Log lines go to a logger installed elsewhere and cannot be observed here. + return; + }; + + assert!( + logs.iter() + .any(|line| line.ends_with("outcome=signed reason=signed")), + "should log the signing outcome: {logs:?}" + ); + for line in &logs { + assert!(!line.contains(TOKEN), "log line leaks the token: {line}"); + assert!( + !line.contains("encypher:v3:") && !line.contains("encypher:policy:v2:"), + "log line leaks the cache key: {line}" + ); + assert!( + !line.contains("ferry"), + "log line leaks article text: {line}" + ); + } + } +} diff --git a/crates/provenance/encypher/src/tests.rs b/crates/provenance/encypher/src/tests.rs new file mode 100644 index 0000000000..a9cbaf0865 --- /dev/null +++ b/crates/provenance/encypher/src/tests.rs @@ -0,0 +1,399 @@ +use super::*; + +use std::sync::Arc; + +use edgezero_core::body::Body as EdgeBody; +use futures::executor::block_on; +use http::{Method, Request, header}; +use serde_json::json; +use trusted_server_core::auction::compile_auction_plan; +use trusted_server_core::auction::orchestrator::AuctionOrchestrator; +use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::ec::EcContext; +use trusted_server_core::html_processor::test_support::page_settings; +use trusted_server_core::integrations::IntegrationRegistry; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use edgezero_core::key_value_store::KvPage; +use trusted_server_core::platform::{ + KvError, PlatformHttpClient, PlatformKvStore, RuntimeServices, +}; +use trusted_server_core::publisher::{ + AppContext, AuctionDispatch, buffer_publisher_response_async, handle_publisher_request, + publisher_response_into_streaming_response, +}; + +const TOKEN: &str = "cwk_test_worker_token_value"; +const MARKER: &str = "\u{E0100}\u{E0101}\u{FE01}"; +const MANIFEST_URL: &str = "https://api.example.com/manifests/doc-1"; +const POLICY_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const ARTICLE: &str = "Harbor Council Votes\ +

Harbor council votes on the ferry schedule

\ +

The harbor council met on Tuesday evening and approved a revised ferry schedule \ + for the coming season.

Residents asked for earlier departures on weekdays.

\ +
Footer
"; + +fn module_settings(table: &str, serve: bool) -> String { + let entry = if serve { + format!("[[serve]]\nmedia_type = \"text/html\"\nmiddleware = [\"{MODULE}\"]\n") + } else { + String::new() + }; + format!("[provenance]\nmodule = \"encypher\"\n\n[provenance.encypher]\n{table}\n{entry}") +} + +fn table() -> String { + format!( + "api_base_url = \"https://api.example.com\"\ndomain = \"publisher.example.com\"\n\ + worker_token = \"{TOKEN}\"\n" + ) +} + +fn placed() -> Settings { + page_settings(&module_settings(&table(), true)) +} + +fn config_value() -> serde_json::Value { + json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": TOKEN, + }) +} + +fn parsed(value: &serde_json::Value) -> Result { + let config: EncypherConfig = + serde_json::from_value(value.clone()).map_err(|error| error.to_string())?; + config.validate().map_err(|error| error.to_string())?; + Ok(config) +} + +#[test] +fn module_constant_is_the_crate_folder() { + assert_eq!( + MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); +} + +#[test] +fn an_unselected_module_registers_nothing() { + let settings = page_settings(""); + + assert!(register(&settings).expect("should read").is_none()); + assert!(!validate(&settings).expect("should read")); +} + +#[test] +fn a_selected_and_placed_module_registers_its_middleware() { + let settings = placed(); + + assert!(validate(&settings).expect("should validate")); + assert!( + register(&settings).expect("should register").is_some(), + "a selected module with a worker token should register" + ); +} + +#[test] +fn a_module_named_in_no_serve_entry_is_refused() { + let settings = page_settings(&module_settings(&table(), false)); + + let error = validate(&settings).expect_err("should refuse the document"); + + assert!( + format!("{error:?}").contains("no [[serve]] entry names"), + "should name the missing entry: {error:?}" + ); +} + +#[test] +fn a_blank_worker_token_registers_nothing() { + for token in ["", " \t"] { + let table = format!( + "api_base_url = \"https://api.example.com\"\ndomain = \"publisher.example.com\"\n\ + worker_token = \"{token}\"\n" + ); + let settings = page_settings(&module_settings(&table, true)); + + assert!( + register(&settings).expect("should read").is_none(), + "token {token:?} should not register" + ); + } +} + +#[test] +fn required_settings_and_bounds_are_enforced() { + for field in ["domain", "api_base_url", "worker_token"] { + let mut config = config_value(); + config + .as_object_mut() + .expect("should be an object") + .remove(field); + assert!(parsed(&config).is_err(), "config without {field} should fail"); + } + let defaults = parsed(&config_value()).expect("should parse defaults"); + assert_eq!(defaults.policy_ttl_seconds, 300); + assert_eq!(defaults.plan_ttl_seconds, 604_800); + assert_eq!(defaults.max_hold_ms, 5_000); + for (field, value) in [ + ("api_base_url", json!("http://api.example.com")), + ("api_base_url", json!("https://api.example.com/?q=1")), + ("domain", json!("Publisher.example.com")), + ("domain", json!("www.publisher.example.com")), + ("domain", json!("publisher.example.com/path")), + ("domain", json!("localhost")), + ("timeout_ms", json!(100)), + ("timeout_ms", json!(20_000)), + ("min_text_length", json!(10)), + ("negative_ttl_seconds", json!(10)), + ("policy_ttl_seconds", json!(301)), + ("article_selector", json!("a".repeat(92))), + ("enabled", json!(true)), + ("path_prefixes", json!(["/news/"])), + ] { + let mut config = config_value(); + config[field] = value.clone(); + assert!(parsed(&config).is_err(), "{field} = {value} should be rejected"); + } + for (max_hold_ms, timeout_ms, valid) in [ + (499, 250, false), + (500, 250, true), + (15_000, 3_000, true), + (15_001, 3_000, false), + (3_000, 3_000, false), + (3_001, 3_000, true), + ] { + let mut config = config_value(); + config["max_hold_ms"] = json!(max_hold_ms); + config["timeout_ms"] = json!(timeout_ms); + assert_eq!( + parsed(&config).is_ok(), + valid, + "max_hold_ms {max_hold_ms} with timeout_ms {timeout_ms}" + ); + } +} + +#[test] +fn excluded_paths_leave_the_middleware_out_of_the_chain() { + let middleware = Encypher { + signer: Arc::new(Signer::new( + parsed(&config_value()).expect("should parse"), + 5_000_000, + )), + }; + + assert!(middleware.handles_path("/news/harbor-vote")); + for path in ["/wp-admin/post", "/feed/", "/sitemap.xml", "/app.js", "/api/v1/x"] { + assert!(!middleware.handles_path(path), "{path} should be left out"); + } + assert_eq!(middleware.phases(), [MiddlewarePhase::Serve]); +} + +/// A runtime KV store held in memory. +#[derive(Default)] +struct MemoryKv(std::sync::Mutex>); + +#[async_trait::async_trait(?Send)] +impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + Ok(self.0.lock().expect("should lock").get(key).cloned()) + } + + async fn put_bytes(&self, key: &str, value: bytes::Bytes) -> Result<(), KvError> { + self.0 + .lock() + .expect("should lock") + .insert(key.to_owned(), value); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: bytes::Bytes, + _ttl: std::time::Duration, + ) -> Result<(), KvError> { + self.put_bytes(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.0.lock().expect("should lock").remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(KvPage::default()) + } +} + +/// An origin answering [`ARTICLE`] with `cache_control`, then the policy and +/// sign answers of the API. +fn stub(cache_control: &str) -> Arc { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + ARTICLE.as_bytes().to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", cache_control), + ], + ); + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + stub.push_response( + 200, + serde_json::to_vec(&json!({ + "success": true, + "policy_digest": POLICY_DIGEST, + "embedding_plan": { + "index_unit": "codepoint", + "operations": [{ "insert_after_index": 5, "marker": MARKER }], + }, + "manifest_url": MANIFEST_URL, + "verification_url": "https://api.example.com/verify/doc-1", + "document_id": "doc-1", + })) + .expect("should encode"), + ); + stub +} + +fn page_request(method: Method) -> Request { + Request::builder() + .method(method) + .uri("https://publisher.example.com/news/harbor-vote?utm_source=feed") + .header(header::HOST, "publisher.example.com") + .body(EdgeBody::empty()) + .expect("should build the page request") +} + +/// The page body a reader receives from the publisher path, through the +/// buffered or the streaming finalizer. +fn served( + settings: &Settings, + services: &RuntimeServices, + request: Request, + streaming: bool, +) -> String { + let registry = IntegrationRegistry::with_registrations(settings, &[builder()]) + .expect("should build the registry"); + let plan = compile_auction_plan(settings).expect("should compile the auction plan"); + let orchestrator = Arc::new(AuctionOrchestrator::from_plan(Arc::new(plan), None)); + let method = request.method().clone(); + block_on(async { + let mut ec_context = EcContext::read_from_request_resolving_geo(settings, &request, services) + .await + .expect("should read the request's state"); + let publisher = handle_publisher_request( + AppContext { + settings, + integration_registry: ®istry, + }, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SurrogateControl, + ) + .await + .expect("should fetch the page"); + let response = if streaming { + publisher_response_into_streaming_response( + publisher, + &method, + Arc::new(settings.clone()), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await + } else { + buffer_publisher_response_async( + publisher, + &method, + settings, + ®istry, + &orchestrator, + services, + ) + .await + } + .expect("should finish the page"); + let body = response + .into_body() + .into_bytes_bounded(16 * 1024 * 1024) + .await + .expect("should read the body"); + String::from_utf8(body.to_vec()).expect("should be UTF-8") + }) +} + +fn services_for(stub: &Arc) -> RuntimeServices { + build_services_with_http_client(Arc::clone(stub) as Arc) + .with_kv_store(Arc::new(MemoryKv::default())) +} + +#[test] +fn the_publisher_path_signs_an_article_through_both_finalizers() { + for streaming in [false, true] { + let stub = stub("public, max-age=60"); + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + streaming, + ); + + assert!( + page.contains(MARKER), + "streaming={streaming}: should embed the marker: {page}" + ); + assert!( + page.contains("c2pa-manifest-url") && page.contains(MANIFEST_URL), + "streaming={streaming}: should write the manifest meta tag" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET", "GET", "POST"], + "streaming={streaming}: origin, policy, then sign" + ); + // The test request carries no TLS signal, so the scheme the publisher + // path detects is `http`. + let policy_uri = &stub.recorded_request_uris()[1]; + assert!( + policy_uri.ends_with("page_url=http%3A%2F%2Fpublisher.example.com%2Fnews%2Fharbor-vote"), + "should send the page address without its query: {policy_uri}" + ); + } +} + +#[test] +fn a_page_the_origin_marked_private_is_never_sent() { + let stub = stub("private, max-age=0"); + + let page = served(&placed(), &services_for(&stub), page_request(Method::GET), false); + + assert!(!page.contains(MARKER), "should not sign a reader's own page"); + assert_eq!( + stub.recorded_request_methods(), + ["GET"], + "should call the origin and nothing else" + ); +} diff --git a/crates/provenance/encypher/src/text.rs b/crates/provenance/encypher/src/text.rs new file mode 100644 index 0000000000..28ebd3450a --- /dev/null +++ b/crates/provenance/encypher/src/text.rs @@ -0,0 +1,500 @@ +use std::sync::LazyLock; + +use regex::Regex; +use sha2::{Digest as _, Sha256}; + +/// Elements whose content canonical text drops: edge-core's set in +/// `integrations/edge-core/fragments.js`. +const TEXT_IGNORED: [&str; 7] = ["script", "style", "noscript", "svg", "math", "code", "pre"]; + +/// Elements whose content browsers never parse as markup (raw text and RCDATA). +/// A `/>` spelling does not close them, and the closing-tag search skips them. +const RAW_TEXT: [&str; 9] = [ + "script", + "style", + "noscript", + "title", + "textarea", + "xmp", + "iframe", + "noembed", + "plaintext", +]; + +static TITLE: LazyLock> = + LazyLock::new(|| Regex::new(r"(?is)]*>(.*?)").ok()); +static DECIMAL_ENTITY: LazyLock> = LazyLock::new(|| Regex::new(r"&#([0-9]+);").ok()); +static HEX_ENTITY: LazyLock> = + LazyLock::new(|| Regex::new(r"&#x([0-9a-fA-F]+);").ok()); + +/// A byte range containing one raw text run in an HTML document. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Fragment { + /// Inclusive UTF-8 byte offset at which the text begins. + pub start: usize, + /// Exclusive UTF-8 byte offset at which the text ends. + pub end: usize, +} + +/// Extracts visible raw-text fragments and their UTF-8 byte ranges. +#[must_use] +pub fn extract_fragments(html: &str) -> Vec { + let mut fragments = Vec::new(); + scan_html( + html, + |name| contains_tag(&TEXT_IGNORED, name), + |start, end| { + if html + .get(start..end) + .is_some_and(|raw| raw.chars().any(|ch| !is_js_whitespace(ch))) + { + fragments.push(Fragment { start, end }); + } + }, + |_, _| {}, + ); + fragments +} + +/// Decodes, normalizes, trims, and joins raw text fragments with single spaces. +#[must_use] +pub fn assemble_text(html: &str, fragments: &[Fragment]) -> String { + let mut output = String::new(); + for fragment in fragments { + if let Some(raw) = html.get(fragment.start..fragment.end) { + let decoded = decode_html_entities(raw); + let normalized = normalize_js_whitespace(&decoded); + if !normalized.is_empty() { + if !output.is_empty() { + output.push(' '); + } + output.push_str(&normalized); + } + } + } + output +} + +/// Returns the canonical visible text of an HTML string. +#[must_use] +pub fn visible_text(html: &str) -> String { + assemble_text(html, &extract_fragments(html)) +} + +/// Returns a lowercase SHA-256 digest of `text` as 64 hexadecimal characters. +#[must_use] +pub fn content_hash_hex(text: &str) -> String { + let digest = Sha256::digest(text.as_bytes()); + let mut output = String::with_capacity(64); + for byte in digest { + if let Some(high) = char::from_digit(u32::from(byte >> 4), 16) { + output.push(high); + } + if let Some(low) = char::from_digit(u32::from(byte & 0x0f), 16) { + output.push(low); + } + } + output +} + +/// Returns at most `max_scalars` Unicode scalar values without splitting one. +#[must_use] +pub fn truncate_scalars(text: &str, max_scalars: usize) -> &str { + let Some((end, _)) = text.char_indices().nth(max_scalars) else { + return text; + }; + text.get(..end).unwrap_or(text) +} + +/// Counts Unicode scalar values in `text`. +#[must_use] +pub fn scalar_count(text: &str) -> usize { + text.chars().count() +} + +/// Extracts the first title, decodes entities, normalizes whitespace, and caps it at 500 scalars. +#[must_use] +pub fn extract_title(html: &str) -> Option { + let regex = TITLE.as_ref()?; + let raw = regex.captures(html)?.get(1)?.as_str(); + let decoded = decode_html_entities(raw); + let mut normalized = normalize_js_whitespace(&decoded); + normalized.truncate(truncate_scalars(&normalized, 500).len()); + Some(normalized) +} + +/// Finds the first closing tag outside comments, the canonical-text ignored +/// elements, and raw-text or RCDATA elements. +#[must_use] +pub fn structural_close_tag(html: &str, tag: &str) -> Option { + let mut found = None; + scan_html( + html, + |name| contains_tag(&TEXT_IGNORED, name) || contains_tag(&RAW_TEXT, name), + |_, _| {}, + |start, token| { + if found.is_none() + && closing_tag_name(token).is_some_and(|name| name.eq_ignore_ascii_case(tag)) + { + found = Some(start); + } + }, + ); + found +} + +/// Scans tags, comments and text runs; both consumers share its comment and +/// quoted-attribute handling. `ignores` picks the elements whose content is +/// skipped: canonical text passes [`TEXT_IGNORED`], and the structural +/// closing-tag search passes [`TEXT_IGNORED`] and [`RAW_TEXT`]. +fn scan_html<'a>( + html: &'a str, + ignores: impl Fn(&str) -> bool, + mut text: impl FnMut(usize, usize), + mut tag: impl FnMut(usize, &'a str), +) { + let bytes = html.as_bytes(); + let mut cursor = 0; + let mut ignored: Option<&str> = None; + while cursor < bytes.len() { + if let Some(name) = ignored { + let Some(open) = find_byte(bytes, b'<', cursor) else { + break; + }; + cursor = open + 1; + if !starts_with_at(bytes, open, b"' || *byte == b'/' + }) + { + continue; + } + let Some(end) = tag_end(bytes, open + 2) else { + break; + }; + // As in browsers, attributes and `/` do not stop an end tag from + // closing. Unlike edge-core's depth count, no other end tag closes. + ignored = None; + cursor = end + 1; + continue; + } + if starts_with_at(bytes, cursor, b"` and `` are whole comments, and `--!>` also ends one. +fn comment_end(bytes: &[u8], start: usize) -> Option { + let body = start + 4; + if starts_with_at(bytes, body, b">") { + return Some(body + 1); + } + if starts_with_at(bytes, body, b"->") { + return Some(body + 2); + } + let dash = find_bytes(bytes, b"-->", body).map(|end| end + 3); + let bang = find_bytes(bytes, b"--!>", body).map(|end| end + 4); + match (dash, bang) { + (Some(dash), Some(bang)) => Some(dash.min(bang)), + (end, None) | (None, end) => end, + } +} + +/// Tokenizer states between a tag's name and its closing `>`. +#[derive(Clone, Copy)] +enum TagState { + Name, + BeforeAttributeName, + AttributeName, + BeforeValue, + Quoted(u8), + Unquoted, +} + +/// Returns the offset of the `>` ending the tag whose name starts at `start`. +/// As in the HTML tokenizer, a quote delimits only a value it begins. +fn tag_end(bytes: &[u8], start: usize) -> Option { + let mut state = TagState::Name; + for (offset, byte) in bytes.get(start..)?.iter().copied().enumerate() { + let space = byte.is_ascii_whitespace(); + state = match state { + TagState::Quoted(quote) => { + if byte == quote { + TagState::BeforeAttributeName + } else { + state + } + } + _ if byte == b'>' => return Some(start + offset), + TagState::Name if space || byte == b'/' => TagState::BeforeAttributeName, + TagState::BeforeAttributeName if !space && byte != b'/' => TagState::AttributeName, + TagState::AttributeName if byte == b'=' => TagState::BeforeValue, + TagState::AttributeName if byte == b'/' => TagState::BeforeAttributeName, + TagState::BeforeValue if byte == b'"' || byte == b'\'' => TagState::Quoted(byte), + TagState::BeforeValue if !space => TagState::Unquoted, + TagState::Unquoted if space => TagState::BeforeAttributeName, + _ => state, + }; + } + None +} + +pub(crate) fn decode_html_entities(input: &str) -> String { + let mut result = input.to_owned(); + for (encoded, decoded) in [ + ("&", "&"), + ("<", "<"), + (">", ">"), + (""", "\""), + ("'", "'"), + ("'", "'"), + ("'", "'"), + ("/", "/"), + (" ", "\u{00a0}"), + ("–", "\u{2013}"), + ("—", "\u{2014}"), + ("‘", "\u{2018}"), + ("’", "\u{2019}"), + ("“", "\u{201c}"), + ("”", "\u{201d}"), + ("…", "\u{2026}"), + ("•", "\u{2022}"), + ] { + if result.contains(encoded) { + result = result.replace(encoded, decoded); + } + } + result = decode_numeric_entities(&result, false); + decode_numeric_entities(&result, true) +} + +pub(crate) fn normalize_js_whitespace(input: &str) -> String { + let mut output = String::with_capacity(input.len()); + let mut pending_space = false; + for ch in input.chars() { + if is_js_whitespace(ch) { + pending_space = !output.is_empty(); + } else { + if pending_space { + output.push(' '); + pending_space = false; + } + output.push(ch); + } + } + output +} + +pub(crate) fn is_js_whitespace(ch: char) -> bool { + matches!( + ch, + '\u{0009}'..='\u{000d}' + | '\u{0020}' + | '\u{00a0}' + | '\u{1680}' + | '\u{2000}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + | '\u{feff}' + ) +} + +fn decode_numeric_entities(input: &str, hexadecimal: bool) -> String { + let regex = if hexadecimal { + &*HEX_ENTITY + } else { + &*DECIMAL_ENTITY + }; + let Some(regex) = regex.as_ref() else { + return input.to_owned(); + }; + let radix = if hexadecimal { 16 } else { 10 }; + regex + .replace_all(input, |captures: ®ex::Captures<'_>| { + captures + .get(1) + .and_then(|digits| u32::from_str_radix(digits.as_str(), radix).ok()) + .and_then(char::from_u32) + .map_or_else( + || { + captures + .get(0) + .map_or("", |matched| matched.as_str()) + .to_owned() + }, + |decoded| decoded.to_string(), + ) + }) + .into_owned() +} + +fn find_byte(bytes: &[u8], needle: u8, start: usize) -> Option { + let relative = bytes + .get(start..)? + .iter() + .position(|byte| *byte == needle)?; + Some(start.saturating_add(relative)) +} + +fn find_bytes(bytes: &[u8], needle: &[u8], start: usize) -> Option { + if needle.is_empty() { + return Some(start.min(bytes.len())); + } + let haystack = bytes.get(start..)?; + let relative = haystack + .windows(needle.len()) + .position(|window| window == needle)?; + Some(start.saturating_add(relative)) +} + +fn starts_with_at(bytes: &[u8], start: usize, needle: &[u8]) -> bool { + bytes + .get(start..) + .is_some_and(|tail| tail.starts_with(needle)) +} + +fn opening_tag_name(token: &str) -> Option<&str> { + let content = token + .strip_prefix('<')? + .trim_start_matches(is_js_whitespace); + if content.starts_with('/') { + return None; + } + take_tag_name(content) +} + +fn closing_tag_name(token: &str) -> Option<&str> { + let content = token.strip_prefix("").then_some(name) +} + +fn take_tag_name(content: &str) -> Option<&str> { + let end = content + .char_indices() + .find(|(_, ch)| !(ch.is_ascii_alphanumeric() || *ch == '_')) + .map_or(content.len(), |(index, _)| index); + (end > 0).then(|| content.get(..end)).flatten() +} + +fn contains_tag(tags: &[&str], name: &str) -> bool { + tags.iter() + .any(|candidate| name.eq_ignore_ascii_case(candidate)) +} + +/// Matches edge-core's `/\/\s*>$/` test on a complete tag token. +fn is_self_closing(token: &str) -> bool { + token + .strip_suffix('>') + .is_some_and(|body| body.trim_end_matches(is_js_whitespace).ends_with('/')) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quoted_markup_and_unmatched_script_tags_stay_inert() { + for html in [ + r#"
Visible
"#, + r#"
Visible
"#, + ] { + assert_eq!(structural_close_tag(html, "head"), html.rfind("")); + assert_eq!(visible_text(html), "Visible"); + } + } + + #[test] + fn browser_end_tag_and_comment_forms_close_where_browsers_do() { + // `` ends the script and `` is a whole comment, so each + // later `` sits inside an attribute value. + for prefix in [ + r#""#, + r#""#, + ] { + let html = format!("{prefix}
Visible
"); + assert_eq!(structural_close_tag(&html, "head"), Some(prefix.len())); + } + assert_eq!( + visible_text("abce" + ); + let unknown = scan_html(&document, None); + assert_eq!( + texts(&document, &unknown), + vec![PNG], + "should edit only the attribute" + ); + assert_eq!( + unknown.refused, + vec![Reason::CspUnknown, Reason::CspUnknown], + "should report both styles under an unseen policy" + ); + + let open = [String::from("img-src 'self' data:")]; + let scan = scan_html(&document, Some(&open)); + assert_eq!( + texts(&document, &scan), + vec![PNG; 3], + "should edit all, in order" + ); + assert!( + scan.found + .windows(2) + .all(|pair| pair[0].start < pair[1].start), + "should keep document order" + ); + + let hashed = [String::from("style-src 'sha256-abc' 'nonce-n'")]; + let scan = scan_html(&document, Some(&hashed)); + assert_eq!( + texts(&document, &scan).len(), + 2, + "should edit the image and the nonced element" + ); + assert_eq!(scan.refused, vec![Reason::CspStyleHash]); + + let meta = format!( + "{document}" + ); + let scan = scan_html(&meta, Some(&open)); + assert_eq!(texts(&meta, &scan), vec![PNG], "should honor a meta policy"); + } + + #[test] + fn only_the_style_s_own_nonce_admits_an_edit_under_a_hash_policy() { + let style = |nonce: &str| { + format!("") + }; + let policy = [String::from("style-src 'sha256-abc' 'NONCE-n0Nce'")]; + for (nonce, edited) in [ + ("n0Nce", true), + ("other", false), + ("n0nce", false), + ("", false), + ] { + let document = style(nonce); + let scan = scan_html(&document, Some(&policy)); + assert_eq!( + scan.found.len(), + usize::from(edited), + "nonce {nonce:?}: should edit only on the exact listed value" + ); + if !edited { + assert_eq!(scan.refused, vec![Reason::CspStyleHash], "nonce {nonce:?}"); + } + } + } + + #[test] + fn style_text_inside_inline_svg_is_markup_whose_references_are_decoded() { + let uri = "data:image/svg+xml,%3Csvg%3E&"; + let open = [String::from("img-src data:")]; + let in_svg = format!( + "" + ); + let scan = scan_html(&in_svg, Some(&open)); + assert!( + scan.found.is_empty(), + "should not rewrite a decoded reference" + ); + assert_eq!(scan.refused, vec![Reason::CharacterReference]); + + let in_html = format!(""); + let scan = scan_html(&in_html, Some(&open)); + assert_eq!( + texts(&in_html, &scan), + vec![uri], + "an HTML style's text is raw, so its `&` is literal" + ); + + let script = format!( + "" + ); + let scan = scan_html(&script, Some(&open)); + assert_eq!( + texts(&script, &scan), + vec![PNG], + "should keep scanning after an SVG script" + ); + } + + #[test] + fn only_url_attributes_are_rewritten() { + let document = format!( + "\ + \"{PNG}\"\ +
" + ); + let scan = scan_html(&document, None); + let found: Vec = scan.found.iter().map(|found| found.start).collect(); + let lazy = document.find("data-lazy-src=\"").expect("lazy") + "data-lazy-src=\"".len(); + let background = document.find("data-bg=\"").expect("bg") + "data-bg=\"".len(); + let object = document.find("data=\"data:").expect("object") + "data=\"".len(); + assert_eq!( + found, + vec![lazy, background, object], + "should rewrite lazy, background and object URLs, never text attributes" + ); + } + + #[test] + fn policies_are_read_from_both_policy_headers() { + let mut headers = http::HeaderMap::new(); + headers.append( + "content-security-policy", + http::HeaderValue::from_static("img-src data:"), + ); + headers.append( + "content-security-policy-report-only", + http::HeaderValue::from_static("style-src 'sha256-abc'"), + ); + assert_eq!( + content_security_policies(&headers), + Some(vec![ + "img-src data:".to_owned(), + "style-src 'sha256-abc'".to_owned() + ]) + ); + } + + #[test] + fn stylesheets_find_url_and_image_set_media_outside_comments() { + let document = format!( + ".a{{background:url({PNG})}}/* url({PNG}) */.b{{background:url( \"{PNG}\" )}}\ + .c{{background-image:image-set(\"{PNG}\" 1x, url({PNG}) 2x)}}\ + .d{{background:url(data:image/png;base64,iVBO\\52w0KGgo=)}}.e{{content:\"{PNG}\"}}" + ); + let scan = scan_stylesheet(&document); + assert_eq!( + texts(&document, &scan), + vec![PNG; 4], + "should find each media URL" + ); + assert_eq!( + scan.refused, + vec![Reason::CssEscape], + "should refuse the escaped URL" + ); + } + + #[test] + fn outermost_inline_svgs_are_found_with_exact_ranges() { + let svg = + ""; + let document = format!("

a

{svg}

b

"); + let scan = scan_html(&document, None); + assert_eq!(scan.svgs.len(), 1, "should find one outermost element"); + let (start, end) = scan.svgs[0]; + assert_eq!(&document[start..end], svg, "should cover the exact bytes"); + } + + #[test] + fn self_closing_svg_attributes_are_scanned_without_whole_asset_signing() { + let document = format!(""); + let scan = scan_html(&document, Some(&[])); + assert_eq!(texts(&document, &scan), [PNG]); + assert!(scan.svgs.is_empty()); + assert!(scan.refused.is_empty()); + } + + #[test] + fn original_svg_provenance_protects_attributes_css_and_nested_svg_ranges() { + for provenance in [ + CARRIER, + "", + "https://example.com/m", + "", + "", + "QUJD", + ] { + let svg = format!( + "{ROOT}{provenance}\ + \ + " + ); + let document = format!("{svg}"); + let scan = scan_html(&document, Some(&[])); + assert_eq!(texts(&document, &scan), [PNG, PNG], "{provenance}"); + assert!(scan.svgs.is_empty(), "must not resign {provenance}"); + assert_eq!( + scan.refused, + [Reason::SvgExistingProvenance], + "{provenance}" + ); + } + } + + #[test] + fn unsigned_svg_descendants_remain_editable_without_a_false_caption_match() { + let document = format!( + "{ROOT}Signed with C2PA\ + \ + " + ); + let scan = scan_html(&document, Some(&[])); + assert_eq!(texts(&document, &scan), [PNG; 3]); + assert_eq!(scan.svgs, [(0, document.len())]); + assert!(scan.refused.is_empty()); + } + + #[test] + fn incomplete_or_uninspectable_svg_ranges_fail_closed_before_child_edits() { + for svg in [ + format!("{ROOT}{CARRIER}"), + format!("{ROOT}"), + format!("{ROOT}

breakout

"), + format!( + "{ROOT}{}", + "".repeat(10_000) + ), + format!(""), + ] { + let document = format!("{svg}"); + let scan = scan_html(&document, Some(&[])); + assert_eq!(texts(&document, &scan), [PNG], "{svg}"); + assert!(scan.svgs.is_empty(), "must not resign {svg}"); + assert_eq!(scan.refused.len(), 1); + } + } + + const ROOT: &str = ""; + const BODY: &str = ""; + const CARRIER: &str = "QUJD+/=="; + + #[test] + fn the_writer_s_carrier_after_the_root_start_tag_keeps_the_svg() { + let original = format!("{ROOT}{BODY}"); + let signed = format!("{ROOT}{CARRIER}{BODY}"); + assert_eq!( + verify_svg_carrier(original.as_bytes(), signed.as_bytes()), + Ok(()), + "should accept the writer's carrier" + ); + let declared = format!("\n{ROOT}{BODY}"); + let signed = format!("\n{ROOT}{CARRIER}{BODY}"); + assert_eq!( + verify_svg_carrier(declared.as_bytes(), signed.as_bytes()), + Ok(()), + "should find the root after a declaration and a comment" + ); + } + + #[test] + fn a_carrier_the_source_held_is_replaced_and_nothing_else() { + let old_element = "T0xE"; + for old in [old_element, ""] { + let original = format!("{ROOT}{old}{BODY}"); + let signed = format!("{ROOT}{CARRIER}{BODY}"); + assert_eq!( + verify_svg_carrier(original.as_bytes(), signed.as_bytes()), + Ok(()), + "should accept replacing {old}" + ); + } + } + + #[test] + fn any_other_change_to_the_svg_is_refused() { + let original = format!("{ROOT}{BODY}"); + let altered = [ + format!("{ROOT}{CARRIER}{BODY}").replace("currentColor", "#000"), + format!("{ROOT}{}{CARRIER}", BODY.replace("", "")) + "", + format!("{ROOT}{BODY}"), + format!("{ROOT}{}{BODY}", CARRIER.replace("QUJD+/==", "QU JD")), + format!("{ROOT} {CARRIER}{BODY}"), + format!("{ROOT}{BODY}"), + original.clone(), + ]; + for signed in altered { + assert_eq!( + verify_svg_carrier(original.as_bytes(), signed.as_bytes()), + Err(Reason::InlineSvgAltered), + "should refuse {signed}" + ); + } + } + + #[test] + fn splice_replaces_exact_ranges_within_budget() { + let document = ""; + let edits = vec![(10, 11, "xx".to_owned()), (23, 24, "y".to_owned())]; + assert_eq!( + splice(document, &edits, 100).as_deref(), + Some("") + ); + assert_eq!( + splice(document, &edits, document.len()), + None, + "should honor the budget" + ); + } +} diff --git a/crates/provenance/encypher/src/lib.rs b/crates/provenance/encypher/src/lib.rs index 4923b1a518..e5ec65909b 100644 --- a/crates/provenance/encypher/src/lib.rs +++ b/crates/provenance/encypher/src/lib.rs @@ -1,4 +1,4 @@ -//! Encypher text provenance for article pages, `provenance.encypher`. +//! Encypher provenance for article pages and their media, `provenance.encypher`. //! //! A serve middleware finds the article in each HTML page a `[[serve]]` entry //! names it for, sends its visible text to the Encypher signing API, and @@ -6,16 +6,36 @@ //! cached in the runtime KV store, bound to a short-lived server policy. //! Every failure passes the page through unchanged. //! +//! Media is signed as its exact bytes through the Encypher media endpoint, +//! under the same policy and worker token: images, audio, video and fonts +//! core captures on the publisher and asset proxy paths, through the +//! module's media signer; media `data:` URIs in the same pages, and their +//! inline SVG as the standalone SVG document proven to parse to the tree the +//! browser built, which is the original bytes where they already are one; +//! and media `data:` URIs in the stylesheets under each `path` a +//! `[[serve]]` entry names `provenance.encypher.stylesheet` for, which the +//! publisher asserts no page loads with `integrity`. Stylesheets under no +//! such path are left as they are. A media item that cannot be signed is +//! delivered as it was and reported unsigned, never as signed. +//! //! The middleware's preparation applies the request gates with no I/O: only //! a `GET` of a `200` page on the configured domain, whose origin did not //! mark it as one reader's, is held. The signing work runs in the stream //! processor's asynchronous finish, which core awaits at the end of the //! document. The module requires no permission and reads no gated value. It -//! sends the visible text of the page's article and the page address without -//! its query. A page whose origin set a cookie or forbade shared caching is -//! never sent, but the module cannot tell a personalized article that carries -//! neither, so a deployment places it only on public article routes and has -//! its origin mark every personalized response private. +//! sends the visible text of the page's article, the page address without +//! its query, and the bytes of the media it signs. A page whose origin set a +//! cookie or forbade shared caching is never sent, but the module cannot +//! tell a personalized article that carries neither, so a deployment places +//! it only on public article routes and has its origin mark every +//! personalized response private. +//! +//! Independently of signing admission, the applicable page middleware protects +//! original provenance-bearing or structurally uncertain inline SVG ranges. +//! Core holds the bounded original before its own rewrites, keeps those ranges +//! opaque through fetch/serve changes and restores their exact bytes before +//! publishing a template or reader response. The same classifier decides both +//! this protection and the module's descendant-edit refusals. use std::any::Any; use std::sync::Arc; @@ -37,12 +57,19 @@ use trusted_server_core::settings::{IntegrationConfig, Settings}; mod api; mod boundary; +mod data_uri; mod embed; +mod embedded; +mod media_api; mod prepared_probe; mod processor; +mod svg_equivalence; mod text; -use processor::{PageRequest, Signer, document_processor, is_excluded_path}; +use processor::{ + EncypherMedia, PageRequest, Signer, document_processor, is_excluded_path, is_excluded_prefix, + stylesheet_processor, +}; /// The id the module's registration and its API backend carry. const INTEGRATION_ID: &str = "provenance_encypher"; @@ -51,6 +78,13 @@ const INTEGRATION_ID: &str = "provenance_encypher"; /// entry names its middleware by, from this crate's folder. pub const MODULE: &str = "provenance.encypher"; +/// The name a `[[serve]]` entry for stylesheets writes, which signs the media +/// `data:` URIs a stylesheet embeds. +pub const STYLESHEET_MIDDLEWARE: &str = "provenance.encypher.stylesheet"; + +/// The media type [`STYLESHEET_MIDDLEWARE`] works on. +const STYLESHEET_MEDIA_TYPE: &str = "text/css"; + /// `[provenance.encypher]`, the module's settings. #[derive(Debug, Clone, Deserialize, Validate)] #[serde(deny_unknown_fields)] @@ -65,11 +99,13 @@ pub struct EncypherConfig { /// The name of the secret-store entry holding the `cwk_` worker /// credential, resolved to the credential itself when settings load. pub worker_token: Redacted, - /// Optional article selector override: `.class`, `[attr=value]` or a tag - /// name. At most 91 characters, so `override:{selector}` fits the API's - /// 100-character `boundary_selector` limit. + /// Optional article selector override: one unescaped `.class`, an exact + /// `[attr=value]` (also single- or double-quoted), or an ASCII tag name. + /// Empty means no override; a valid selector absent from a page keeps + /// the detection fallback. At most 91 characters, so + /// `override:{selector}` fits the API's 100-character limit. #[serde(default)] - #[validate(length(max = 91))] + #[validate(length(max = 91), custom(function = validate_article_selector))] pub article_selector: String, /// Minimum visible article text, in Unicode scalars. #[serde(default = "default_min_text_length")] @@ -102,8 +138,44 @@ pub struct EncypherConfig { #[serde(default = "default_negative_ttl_seconds")] #[validate(range(min = 60, max = 86_400))] pub negative_ttl_seconds: u64, + /// Largest media item signed, in bytes: a captured image or a decoded + /// embedded item. Larger media is delivered unsigned. Bounded by what the + /// endpoint embeds as source data when the source has no safe address. + #[serde(default = "default_max_media_bytes")] + #[validate(range(min = 1024, max = 8_000_000))] + pub max_media_bytes: usize, + /// Maximum uncached media-signing calls in one HTML or stylesheet pass. + /// Cached assets do not consume calls. The document's existing time and + /// output bounds still apply independently. + #[serde(default = "default_max_media_sign_calls_per_document")] + #[validate(range(min = 1, max = 128))] + pub max_media_sign_calls_per_document: usize, + /// Largest media signed through the prepared lane, in bytes: hashed in + /// bounded chunks where it is delivered, never uploaded or held whole, + /// and given a remote manifest its bytes are left unchanged by. `0`, the + /// default, keeps the lane off, so no deployment starts hashing large + /// files on its own; turning it on also needs the signed probe release + /// and the prepared endpoints. Otherwise at least 16 MiB, and at most the + /// prepared service's largest source. Media between `max_media_bytes` + /// and this is the lane's. A bounded example is `1073741824` (1 GiB) + /// with `max_large_media_ms = 60000`. + #[serde(default = "default_max_large_media_bytes")] + #[validate(custom(function = validate_max_large_media_bytes))] + pub max_large_media_bytes: u64, + /// Longest one large source may take, in milliseconds: core's hash of + /// the whole source and the module's probe and API calls together. A + /// source not signed in time is delivered unsigned. + #[serde(default = "default_max_large_media_ms")] + #[validate(range(min = 5_000, max = 600_000))] + pub max_large_media_ms: u64, } +/// The prepared service's largest source, in bytes: the largest integer a +/// JSON number carries exactly, less the room it keeps for a manifest. +const MAX_PREPARED_SOURCE_BYTES: u64 = 9_007_199_246_352_383; +/// The smallest non-zero `max_large_media_bytes`, above every captured size. +const MIN_LARGE_MEDIA_BYTES: u64 = 16 * 1024 * 1024; + impl IntegrationConfig for EncypherConfig {} fn default_min_text_length() -> usize { @@ -134,6 +206,38 @@ fn default_negative_ttl_seconds() -> u64 { 300 } +fn default_max_media_bytes() -> usize { + 5_000_000 +} + +fn default_max_media_sign_calls_per_document() -> usize { + 8 +} + +fn default_max_large_media_bytes() -> u64 { + 0 +} + +fn default_max_large_media_ms() -> u64 { + 60_000 +} + +fn validate_max_large_media_bytes(value: u64) -> Result<(), ValidationError> { + if value == 0 || (MIN_LARGE_MEDIA_BYTES..=MAX_PREPARED_SOURCE_BYTES).contains(&value) { + Ok(()) + } else { + Err(ValidationError::new("encypher_max_large_media_bytes")) + } +} + +fn validate_article_selector(value: &str) -> Result<(), ValidationError> { + if value.is_empty() || boundary::is_valid_selector(value) { + Ok(()) + } else { + Err(ValidationError::new("encypher_article_selector")) + } +} + fn validate_api_base_url(value: &str) -> Result<(), ValidationError> { let parsed = url::Url::parse(value).map_err(|_| ValidationError::new("encypher_api_base_url"))?; @@ -187,9 +291,16 @@ fn always_in_use(_table: &serde_json::Map) -> bool { true } -/// The page change the module supplies, as a `[[serve]]` entry names it. -const SUPPLIED_MIDDLEWARE: &[SuppliedMiddleware] = - &[SuppliedMiddleware::on_html(MODULE, MiddlewarePhase::Serve)]; +/// The changes the module supplies, as `[[serve]]` entries name them: the +/// page's article and embedded media, and a stylesheet's embedded media. +const SUPPLIED_MIDDLEWARE: &[SuppliedMiddleware] = &[ + SuppliedMiddleware::on_html(MODULE, MiddlewarePhase::Serve), + SuppliedMiddleware::on( + STYLESHEET_MIDDLEWARE, + MiddlewarePhase::Serve, + STYLESHEET_MEDIA_TYPE, + ), +]; /// The builder a deployment installs to offer this module. #[must_use] @@ -220,12 +331,47 @@ fn require_an_entry(settings: &Settings) -> Result<(), Report Result<(), Report> { + let unscoped = settings.serve.entries().iter().any(|entry| { + entry + .middleware + .iter() + .any(|name| name == STYLESHEET_MIDDLEWARE) + && entry + .path + .as_deref() + .is_none_or(|path| path.is_empty() || path == "/") + }); + if !unscoped { + return Ok(()); + } + Err(Report::new(TrustedServerError::Settings { + message: format!( + "a [[serve]] entry names `{STYLESHEET_MIDDLEWARE}` for every path. Signing a \ + stylesheet's embedded media changes its bytes and breaks a page that loads it \ + with `integrity`, so give the entry a `path` for the stylesheets no page loads \ + with `integrity`" + ), + })) +} + /// The module's settings when a section selects it. fn config(settings: &Settings) -> Result, Report> { let Some(config) = settings.module_config::(MODULE)? else { return Ok(None); }; require_an_entry(settings)?; + require_scoped_stylesheet_entries(settings)?; Ok(Some(config)) } @@ -240,8 +386,8 @@ pub fn validate(settings: &Settings) -> Result> Ok(config(settings)?.is_some()) } -/// Registers the middleware when a section selects the module and its worker -/// token is not blank. +/// Registers the middleware and the media signer when a section selects the +/// module and its worker token is not blank. /// /// # Errors /// @@ -260,17 +406,23 @@ pub fn register( let max_document_bytes = config .max_document_bytes .min(settings.publisher.max_buffered_body_bytes); + let signer = Arc::new(Signer::new(config, max_document_bytes)); Ok(Some( IntegrationRegistration::builder(INTEGRATION_ID) .without_js() .with_middleware(Arc::new(Encypher { - signer: Arc::new(Signer::new(config, max_document_bytes)), + signer: Arc::clone(&signer), })) + .with_middleware(Arc::new(EncypherStylesheet { + signer: Arc::clone(&signer), + })) + .with_media_signer(Arc::new(EncypherMedia::new(signer))) .build(), )) } -/// The middleware, which signs the article of each admitted page. +/// The middleware, which signs the article and the embedded media of each +/// admitted page. pub struct Encypher { signer: Arc, } @@ -293,6 +445,10 @@ impl Middleware for Encypher { media_type == HTML_MEDIA_TYPE } + fn source_protection(&self) -> Option { + Some(embedded::source_protection) + } + /// Every path but an API, feed, admin or well-known path, or an asset, /// which is never an article. fn handles_path(&self, path: &str) -> bool { @@ -311,38 +467,112 @@ impl Middleware for Encypher { call: ModuleCall<'a>, context: &'a MiddlewareContext<'a>, ) -> Option> { - let admission = admit(&self.signer, &call, context); + let admission = admit(&call, context, |page| self.signer.admit(page)); Some(Box::pin(async move { Ok(Box::new(admission) as Box) })) } + /// Holds the admitted page, with the security policies its response + /// carries, which decide whether inline style media may be edited. fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { let Some(page) = context.take_prepared::(MODULE).flatten() else { return MiddlewareAction::pass(); }; + let policies = context + .response_headers + .and_then(embedded::content_security_policies); + MiddlewareAction { + stream: Some(Box::new(document_processor(&self.signer, page, policies))), + ..MiddlewareAction::pass() + } + } +} + +/// The stylesheet middleware, which signs the media `data:` URIs of each +/// admitted stylesheet. +pub struct EncypherStylesheet { + signer: Arc, +} + +impl std::fmt::Debug for EncypherStylesheet { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("EncypherStylesheet") + .finish_non_exhaustive() + } +} + +impl Middleware for EncypherStylesheet { + fn middleware_id(&self) -> &'static str { + STYLESHEET_MIDDLEWARE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == STYLESHEET_MEDIA_TYPE + } + + /// Every path but an API, feed, admin or well-known path. A `.css` + /// suffix is what a stylesheet has. + fn handles_path(&self, path: &str) -> bool { + !is_excluded_prefix(path) + } + + /// Serve only, as for pages. + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + /// Applies the request gates, with no I/O, and keeps the stylesheet to + /// sign. + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + let admission = admit(&call, context, |page| self.signer.admit_stylesheet(page)); + Some(Box::pin(async move { + Ok(Box::new(admission) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let Some(page) = context + .take_prepared::(STYLESHEET_MIDDLEWARE) + .flatten() + else { + return MiddlewareAction::pass(); + }; MiddlewareAction { - stream: Some(Box::new(document_processor(&self.signer, page))), + stream: Some(Box::new(stylesheet_processor(&self.signer, page))), ..MiddlewareAction::pass() } } } -/// The page a document is the body of, when the gates admit it. -fn admit(signer: &Signer, call: &ModuleCall<'_>, context: &MiddlewareContext<'_>) -> Admission { +/// The document a request is for, when `gate` admits it. +fn admit( + call: &ModuleCall<'_>, + context: &MiddlewareContext<'_>, + gate: impl FnOnce(PageRequest) -> Result, +) -> Admission { let request = ModuleRequest::from_module_context(call).ok()?; + let removed_prefix = request.removed_prefix().unwrap_or_default(); let page = PageRequest { method: request.method().clone(), + // The address the reader asked for: a host that serves several + // publishers by path signs each page for its own address. page_url: format!( - "{}://{}{}", + "{}://{}{removed_prefix}{}", request.scheme(), request.host(), request.path() ), + removed_prefix: removed_prefix.to_owned(), response_status: context.status.as_u16(), origin_response_is_private: context.origin_private, }; - match signer.admit(page) { + match gate(page) { Ok(page) => Some(page), Err(gate) => { log::debug!("encypher: document not held, gate={}", gate.code()); diff --git a/crates/provenance/encypher/src/media_api.rs b/crates/provenance/encypher/src/media_api.rs new file mode 100644 index 0000000000..2d8d7de9aa --- /dev/null +++ b/crates/provenance/encypher/src/media_api.rs @@ -0,0 +1,831 @@ +//! Wire contract of the Encypher media signing endpoint, +//! `POST /api/v1/public/cdn/media/sign`. +//! +//! The request is `multipart/form-data` with a `metadata` JSON part and the +//! exact `source` bytes. Neither core nor this module transforms media, so no +//! `final` part is sent and the execution record lists no operations. A `200` +//! answer is the exact signed bytes, described by `X-Encypher-*` headers that +//! [`verify_signed_response`] checks before any byte reaches a reader. + +use bytes::{BufMut as _, Bytes, BytesMut}; +use http::{HeaderMap, HeaderValue}; +use serde::ser::SerializeSeq as _; +use serde::{Deserialize, Serialize, Serializer}; +use sha2::{Digest as _, Sha256}; +use trusted_server_core::middleware::media_type_of; + +use crate::api::valid_policy_digest; + +/// The media signing route, below the API origin. +pub(crate) const MEDIA_SIGN_PATH: &str = "/api/v1/public/cdn/media/sign"; +/// The request header carrying the replay key the endpoint requires. +pub(crate) const IDEMPOTENCY_KEY_HEADER: &str = "idempotency-key"; +/// The whole multipart body the endpoint accepts. +pub(crate) const MAX_REQUEST_BYTES: usize = 100_000_000; +/// What signing may add to the bytes: the manifest store with its +/// certificate chain and the embedded source data. +pub(crate) const MAX_MANIFEST_OVERHEAD_BYTES: usize = 10 * 1024 * 1024; +/// A failure body is only read for its code. +pub(crate) const MAX_ERROR_BODY_BYTES: usize = 4096; +/// Longest manifest instance or document identifier accepted. +const MAX_IDENTIFIER_BYTES: usize = 256; +/// Longest URL the endpoint accepts or returns. +const MAX_URL_BYTES: usize = 2048; +/// Longest failure code kept for the log. +const MAX_ERROR_CODE_BYTES: usize = 64; + +const POLICY_DIGEST_HEADER: &str = "x-encypher-policy-digest"; +const SIGNED_SHA256_HEADER: &str = "x-encypher-signed-sha256"; +const INSTANCE_ID_HEADER: &str = "x-encypher-manifest-instance-id"; +const DISPOSITION_HEADER: &str = "x-encypher-manifest-disposition"; +const MANIFEST_URL_HEADER: &str = "x-encypher-manifest-url"; +const MANIFEST_SHA256_HEADER: &str = "x-encypher-manifest-sha256"; +const LINK_REQUIRED_HEADER: &str = "x-encypher-manifest-link-required"; +const VERIFICATION_URL_HEADER: &str = "x-encypher-verification-url"; +const DOCUMENT_ID_HEADER: &str = "x-encypher-document-id"; + +/// Every signing header the endpoint answers with, in the order a cached +/// answer stores them. +pub(crate) const SIGNING_HEADERS: [&str; 10] = [ + "content-type", + POLICY_DIGEST_HEADER, + SIGNED_SHA256_HEADER, + INSTANCE_ID_HEADER, + DISPOSITION_HEADER, + MANIFEST_URL_HEADER, + MANIFEST_SHA256_HEADER, + LINK_REQUIRED_HEADER, + VERIFICATION_URL_HEADER, + DOCUMENT_ID_HEADER, +]; + +/// The `metadata` part of one media signing request. +#[derive(Debug, Serialize)] +pub(crate) struct MediaMetadata<'a> { + /// The reader-facing absolute URL the media is delivered on or in. + pub(crate) page_url: &'a str, + /// The server policy the edge holds, which the endpoint refuses when + /// stale. + pub(crate) expected_policy_digest: &'a str, + pub(crate) source: SourceMetadata<'a>, + pub(crate) execution_record: ExecutionRecord<'a>, + pub(crate) delivery: Delivery, +} + +/// What the edge says about the captured source bytes. +#[derive(Debug, Serialize)] +pub(crate) struct SourceMetadata<'a> { + /// A canonical `https` address of the source, or `null` where there is + /// none: embedded media, or an address with a query or credentials, + /// which is never stripped to make one. Without it the endpoint binds the + /// source by embedding its bytes. + pub(crate) reference: Option<&'a str>, + /// The exact hostname the source was fetched from, which the endpoint + /// holds the reference and the claim's managed origins against, or + /// `null` only for media embedded in a document, which was fetched from + /// no host of its own. Never a scheme, port, path or query. + pub(crate) origin_host: Option<&'a str>, + /// The declared media type, a hint the endpoint checks against the bytes. + pub(crate) mime: &'a str, + pub(crate) sha256: &'a str, + pub(crate) byte_length: usize, + /// The source is a platform image optimizer's answer: the endpoint + /// refuses it unless the bytes carry an embedded manifest, which it + /// validates as it validates any ingredient's. Sent only when true. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub(crate) require_embedded_manifest: bool, +} + +/// What the edge did to the source before it was delivered: nothing. +#[derive(Debug, Serialize)] +pub(crate) struct ExecutionRecord<'a> { + pub(crate) source_sha256: &'a str, + pub(crate) final_sha256: &'a str, + /// The exact source was captured and no operation was applied. + pub(crate) complete: bool, + pub(crate) operations: NoOperations, +} + +/// The empty operation list of media delivered as it was captured. +#[derive(Debug)] +pub(crate) struct NoOperations; + +impl Serialize for NoOperations { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_seq(Some(0))?.end() + } +} + +/// How the signed representation reaches readers. +#[derive(Debug, Serialize)] +pub(crate) struct Delivery { + /// Whether every response of the representation can carry a `Link` + /// header naming a remote manifest. + pub(crate) http_link_header: bool, +} + +impl<'a> MediaMetadata<'a> { + /// Metadata for `source` delivered unmodified: no `final` part, an empty + /// and complete execution record. + pub(crate) fn unmodified( + page_url: &'a str, + expected_policy_digest: &'a str, + source: SourceMetadata<'a>, + http_link_header: bool, + ) -> Self { + let sha256 = source.sha256; + Self { + page_url, + expected_policy_digest, + source, + execution_record: ExecutionRecord { + source_sha256: sha256, + final_sha256: sha256, + complete: true, + operations: NoOperations, + }, + delivery: Delivery { http_link_header }, + } + } +} + +/// A built multipart request body and its `Content-Type`. +#[derive(Debug)] +pub(crate) struct Multipart { + pub(crate) content_type: String, + pub(crate) body: Bytes, +} + +/// Builds the `multipart/form-data` body with the `metadata` and `source` +/// parts, with a boundary that occurs in neither. +pub(crate) fn multipart(metadata: &[u8], source: &[u8]) -> Multipart { + let seed = sha256_hex(metadata); + let mut attempt = 0_u32; + let boundary = loop { + let candidate = format!( + "encypher-media-{}-{attempt}", + seed.get(..32).unwrap_or(&seed) + ); + let delimiter = format!("--{candidate}"); + if !contains(metadata, delimiter.as_bytes()) && !contains(source, delimiter.as_bytes()) { + break candidate; + } + attempt = attempt.saturating_add(1); + }; + let metadata_head = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"metadata\"\r\n\ + Content-Type: application/json\r\n\r\n" + ); + let source_head = format!( + "\r\n--{boundary}\r\nContent-Disposition: form-data; name=\"source\"; \ + filename=\"source\"\r\nContent-Type: application/octet-stream\r\n\r\n" + ); + let closing = format!("\r\n--{boundary}--\r\n"); + let mut body = BytesMut::with_capacity( + metadata_head.len() + metadata.len() + source_head.len() + source.len() + closing.len(), + ); + body.put_slice(metadata_head.as_bytes()); + body.put_slice(metadata); + body.put_slice(source_head.as_bytes()); + body.put_slice(source); + body.put_slice(closing.as_bytes()); + Multipart { + content_type: format!("multipart/form-data; boundary={boundary}"), + body: body.freeze(), + } +} + +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +/// Lowercase hex SHA-256 of `bytes`. +pub(crate) fn sha256_hex(bytes: &[u8]) -> String { + hex_lower(&Sha256::digest(bytes)) +} + +fn hex_lower(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(char::from(DIGITS[usize::from(byte >> 4)])); + out.push(char::from(DIGITS[usize::from(byte & 0x0f)])); + } + out +} + +/// The bare media type an answer must carry for media declared as `declared`, +/// with the common `image/jpg` spelling read as `image/jpeg`, and the +/// `audio/x-aac` and `audio/aacp` spellings of raw AAC as `audio/aac`. +pub(crate) fn canonical_media_type(declared: &str) -> String { + let media_type = media_type_of(declared); + match media_type.as_str() { + "image/jpg" => "image/jpeg".to_owned(), + "audio/x-aac" | "audio/aacp" => "audio/aac".to_owned(), + _ => media_type, + } +} + +/// Whether `reference` may be sent as the source's address: canonical +/// `https` on the default port, with a path, no credentials, query or +/// fragment, on `host`. Anything else is sent as no reference. +pub(crate) fn safe_source_reference(reference: &str, host: &str) -> bool { + if reference.len() > MAX_URL_BYTES || !reference.is_ascii() { + return false; + } + let Ok(parsed) = url::Url::parse(reference) else { + return false; + }; + parsed.scheme() == "https" + && parsed.as_str() == reference + && parsed.username().is_empty() + && parsed.password().is_none() + && parsed.port().is_none() + && parsed.query().is_none() + && parsed.fragment().is_none() + && parsed.path() != "/" + && parsed.host_str() == Some(host) +} + +/// Whether `host` is one canonical hostname alone: no scheme, user +/// information, port, path or query, in lower case. +pub(crate) fn valid_origin_host(host: &str) -> bool { + !host.is_empty() + && host.len() <= 253 + && url::Url::parse(&format!("https://{host}/")).is_ok_and(|parsed| { + parsed.host_str() == Some(host) + && parsed.port().is_none() + && parsed.username().is_empty() + && parsed.path() == "/" + }) +} + +/// Where the signed bytes' manifest lives. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Disposition { + /// Inside the signed bytes. + Embedded, + /// At [`SignedMedia::manifest_url`], named by the bytes or a `Link`. + External, +} + +/// What a signing answer must match. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Expected<'a> { + pub(crate) policy_digest: &'a str, + /// The bare media type the source was delivered as. + pub(crate) media_type: &'a str, + /// SHA-256 of the bytes as they were before signing. + pub(crate) unsigned_sha256: &'a str, + /// Whether every response of the representation can carry a `Link`. + pub(crate) link_deliverable: bool, +} + +/// A verified signing answer. +#[derive(Debug, Clone)] +pub(crate) struct SignedMedia { + pub(crate) bytes: Bytes, + pub(crate) content_type: HeaderValue, + pub(crate) instance_id: String, + pub(crate) document_id: String, + pub(crate) verification_url: String, + pub(crate) disposition: Disposition, + pub(crate) manifest_url: Option, +} + +impl SignedMedia { + /// The `Link` header value naming a remote manifest, when there is one. + pub(crate) fn link(&self) -> Option { + self.manifest_url + .as_deref() + .map(|url| format!("<{url}>; rel=\"c2pa-manifest\"")) + } +} + +/// Why a `200` answer is not accepted as signed media. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ResponseError { + /// The answer's media type is not the one the source was delivered as. + ContentType, + /// The answer was signed under another policy. + PolicyDigest, + /// The body is not the bytes the signed digest names. + SignedDigest, + /// A header is missing, repeated or malformed. + Malformed, + /// The manifest is remote and needs a `Link` the delivery cannot carry. + LinkUndeliverable, + /// The body carries no manifest and no reference to one. + Unsigned, +} + +/// The one value of `name`, when it appears exactly once as visible ASCII. +fn single<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + let mut values = headers.get_all(name).iter(); + let value = values.next()?; + if values.next().is_some() { + return None; + } + value.to_str().ok() +} + +fn valid_identifier(value: &str) -> bool { + !value.is_empty() + && value.len() <= MAX_IDENTIFIER_BYTES + && value.bytes().all(|byte| byte.is_ascii_graphic()) +} + +/// Whether `value` is a canonical absolute `https` URL that is safe inside a +/// `Link` header's angle brackets. +fn valid_https_url(value: &str) -> bool { + value.len() <= MAX_URL_BYTES + && value + .bytes() + .all(|byte| byte.is_ascii_graphic() && !matches!(byte, b'<' | b'>' | b'"')) + && url::Url::parse(value).is_ok_and(|parsed| { + parsed.scheme() == "https" && parsed.host_str().is_some() && parsed.as_str() == value + }) +} + +/// Checks a `200` signing answer against what was sent, and keeps it only +/// when it is signed media this delivery can carry. +/// +/// # Errors +/// +/// Returns the first [`ResponseError`] the answer shows. +pub(crate) fn verify_signed_response( + headers: &HeaderMap, + body: Bytes, + expected: &Expected<'_>, +) -> Result { + let content_type = single(headers, "content-type").ok_or(ResponseError::Malformed)?; + if canonical_media_type(content_type) != expected.media_type { + return Err(ResponseError::ContentType); + } + let policy_digest = single(headers, POLICY_DIGEST_HEADER).ok_or(ResponseError::Malformed)?; + if !valid_policy_digest(policy_digest) { + return Err(ResponseError::Malformed); + } + if policy_digest != expected.policy_digest { + return Err(ResponseError::PolicyDigest); + } + let signed_sha256 = single(headers, SIGNED_SHA256_HEADER).ok_or(ResponseError::Malformed)?; + if !valid_policy_digest(signed_sha256) { + return Err(ResponseError::Malformed); + } + if body.is_empty() || sha256_hex(&body) != signed_sha256 { + return Err(ResponseError::SignedDigest); + } + let instance_id = single(headers, INSTANCE_ID_HEADER) + .filter(|value| valid_identifier(value)) + .ok_or(ResponseError::Malformed)?; + let document_id = single(headers, DOCUMENT_ID_HEADER) + .filter(|value| valid_identifier(value)) + .ok_or(ResponseError::Malformed)?; + let verification_url = single(headers, VERIFICATION_URL_HEADER) + .filter(|value| valid_https_url(value)) + .ok_or(ResponseError::Malformed)?; + let link_required = match single(headers, LINK_REQUIRED_HEADER) { + Some("true") => true, + Some("false") => false, + _ => return Err(ResponseError::Malformed), + }; + let manifest_url = headers.get(MANIFEST_URL_HEADER).map(|_| { + single(headers, MANIFEST_URL_HEADER) + .filter(|value| valid_https_url(value)) + .ok_or(ResponseError::Malformed) + }); + let changed = signed_sha256 != expected.unsigned_sha256; + let disposition = match single(headers, DISPOSITION_HEADER) { + Some("embedded") => { + // The manifest store is in the bytes, so they changed, hold the + // `c2pa` label and point nowhere else. These are consistency + // checks against a wrong answer, not a signature check: the + // signature is trusted through the authenticated endpoint and + // the digest binding above. + if manifest_url.is_some() || link_required { + return Err(ResponseError::Malformed); + } + if !changed || !contains(&body, b"c2pa") { + return Err(ResponseError::Unsigned); + } + Disposition::Embedded + } + Some("external") => { + if manifest_url.is_none() { + return Err(ResponseError::Malformed); + } + single(headers, MANIFEST_SHA256_HEADER) + .filter(|value| valid_policy_digest(value)) + .ok_or(ResponseError::Malformed)?; + if link_required && !expected.link_deliverable { + return Err(ResponseError::LinkUndeliverable); + } + // A remote manifest no `Link` names must be named by the bytes. + if !link_required && !changed { + return Err(ResponseError::Unsigned); + } + Disposition::External + } + _ => return Err(ResponseError::Malformed), + }; + let manifest_url = manifest_url.transpose()?.map(str::to_owned); + let content_type = HeaderValue::from_str(content_type).map_err(|_| ResponseError::Malformed)?; + Ok(SignedMedia { + bytes: body, + content_type, + instance_id: instance_id.to_owned(), + document_id: document_id.to_owned(), + verification_url: verification_url.to_owned(), + disposition, + manifest_url, + }) +} + +/// The failure code of an error answer, kept only as a short identifier. +pub(crate) fn error_code(body: &[u8]) -> Option { + #[derive(Deserialize)] + struct ErrorBody { + error: ErrorDetail, + } + #[derive(Deserialize)] + struct ErrorDetail { + code: String, + } + if body.len() > MAX_ERROR_BODY_BYTES { + return None; + } + let parsed: ErrorBody = serde_json::from_slice(body).ok()?; + let code = parsed.error.code; + (!code.is_empty() + && code.len() <= MAX_ERROR_CODE_BYTES + && code + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')) + .then_some(code) +} + +#[cfg(test)] +mod tests { + use http::HeaderName; + use serde_json::json; + + use super::*; + + const DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const UNSIGNED: &[u8] = b"\x89PNG\r\n\x1a\nunsigned"; + const SIGNED: &[u8] = b"\x89PNG\r\n\x1a\nunsigned-caBXjumbc2pa-manifest"; + + fn expected(unsigned_sha256: &str, link_deliverable: bool) -> Expected<'_> { + Expected { + policy_digest: DIGEST, + media_type: "image/png", + unsigned_sha256, + link_deliverable, + } + } + + fn embedded_headers(body: &[u8]) -> HeaderMap { + let mut headers = HeaderMap::new(); + for (name, value) in [ + ("content-type", "image/png".to_owned()), + (POLICY_DIGEST_HEADER, DIGEST.to_owned()), + (SIGNED_SHA256_HEADER, sha256_hex(body)), + (INSTANCE_ID_HEADER, "xmp:iid:0001".to_owned()), + (DISPOSITION_HEADER, "embedded".to_owned()), + (LINK_REQUIRED_HEADER, "false".to_owned()), + ( + VERIFICATION_URL_HEADER, + "https://verify.example.com/doc-1".to_owned(), + ), + (DOCUMENT_ID_HEADER, "doc-1".to_owned()), + ] { + headers.insert( + HeaderName::from_static(name), + HeaderValue::from_str(&value).expect("should build header"), + ); + } + headers + } + + fn external_headers(body: &[u8], link_required: bool) -> HeaderMap { + let mut headers = embedded_headers(body); + headers.insert(DISPOSITION_HEADER, HeaderValue::from_static("external")); + headers.insert( + MANIFEST_URL_HEADER, + HeaderValue::from_static("https://api.example.com/manifests/doc-1"), + ); + headers.insert( + MANIFEST_SHA256_HEADER, + HeaderValue::from_str(DIGEST).expect("should build header"), + ); + headers.insert( + LINK_REQUIRED_HEADER, + HeaderValue::from_static(if link_required { "true" } else { "false" }), + ); + headers + } + + #[test] + fn metadata_matches_the_frozen_wire_shape() { + let sha = sha256_hex(UNSIGNED); + let metadata = MediaMetadata::unmodified( + "https://publisher.example.com/news/story", + DIGEST, + SourceMetadata { + reference: None, + origin_host: None, + mime: "image/png", + sha256: &sha, + byte_length: UNSIGNED.len(), + require_embedded_manifest: false, + }, + false, + ); + let value = serde_json::to_value(&metadata).expect("should serialize metadata"); + assert_eq!( + value, + json!({ + "page_url": "https://publisher.example.com/news/story", + "expected_policy_digest": DIGEST, + "source": { + "reference": null, + "origin_host": null, + "mime": "image/png", + "sha256": sha, + "byte_length": UNSIGNED.len(), + }, + "execution_record": { + "source_sha256": sha, + "final_sha256": sha, + "complete": true, + "operations": [], + }, + "delivery": { "http_link_header": false }, + }), + "should send an unmodified, complete record with a null reference" + ); + } + + #[test] + fn an_optimizer_answer_requires_its_embedded_manifest_on_the_wire() { + let sha = sha256_hex(SIGNED); + let metadata = MediaMetadata::unmodified( + "https://publisher.example.com/images/photo.jpg", + DIGEST, + SourceMetadata { + reference: None, + origin_host: Some("origin.example.com"), + mime: "image/png", + sha256: &sha, + byte_length: SIGNED.len(), + require_embedded_manifest: true, + }, + true, + ); + let value = serde_json::to_value(&metadata).expect("should serialize metadata"); + assert_eq!( + value["source"], + json!({ + "reference": null, + "origin_host": "origin.example.com", + "mime": "image/png", + "sha256": sha, + "byte_length": SIGNED.len(), + "require_embedded_manifest": true, + }), + "should require the optimizer's embedded manifest and name no address" + ); + assert_eq!( + value["execution_record"], + json!({ + "source_sha256": sha, + "final_sha256": sha, + "complete": true, + "operations": [], + }), + "should report no operation of the edge's own" + ); + } + + #[test] + fn multipart_carries_exact_source_bytes_under_an_absent_boundary() { + let source = b"--encypher-media-binary\r\n\x00\xff"; + let built = multipart(br#"{"page_url":"x"}"#, source); + let boundary = built + .content_type + .strip_prefix("multipart/form-data; boundary=") + .expect("should declare a boundary"); + let body = built.body.as_ref(); + let source_start = body + .windows(4) + .position(|window| window == b"\r\n\r\n") + .and_then(|first| { + body[first + 4..] + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|second| first + 4 + second + 4) + }) + .expect("should find the source part body"); + let tail = format!("\r\n--{boundary}--\r\n"); + assert!(body.ends_with(tail.as_bytes()), "should close the body"); + assert_eq!( + &body[source_start..body.len() - tail.len()], + source, + "should carry the exact source bytes" + ); + assert!( + body.windows(9).any(|window| window == b"name=\"sou"), + "should name the source part" + ); + assert!( + !body.windows(7).any(|window| window == b"\"final\""), + "should send no final part for unmodified media" + ); + } + + #[test] + fn an_embedded_answer_is_accepted_only_with_matching_bytes_and_policy() { + let unsigned = sha256_hex(UNSIGNED); + let signed = verify_signed_response( + &embedded_headers(SIGNED), + Bytes::from_static(SIGNED), + &expected(&unsigned, false), + ) + .expect("should accept a verified embedded answer"); + assert_eq!(signed.bytes.as_ref(), SIGNED, "should keep the exact bytes"); + assert_eq!( + signed.disposition, + Disposition::Embedded, + "should be embedded" + ); + assert_eq!(signed.link(), None, "should need no Link"); + + let tampered = verify_signed_response( + &embedded_headers(SIGNED), + Bytes::from_static(b"\x89PNG\r\n\x1a\nc2pa-other"), + &expected(&unsigned, false), + ); + assert_eq!( + tampered.expect_err("should refuse a body the digest does not name"), + ResponseError::SignedDigest + ); + + let mut stale = embedded_headers(SIGNED); + stale.insert( + POLICY_DIGEST_HEADER, + HeaderValue::from_str(&"b".repeat(64)).expect("should build header"), + ); + assert_eq!( + verify_signed_response( + &stale, + Bytes::from_static(SIGNED), + &expected(&unsigned, false) + ) + .expect_err("should refuse another policy"), + ResponseError::PolicyDigest + ); + + let mut retyped = embedded_headers(SIGNED); + retyped.insert("content-type", HeaderValue::from_static("image/jpeg")); + assert_eq!( + verify_signed_response( + &retyped, + Bytes::from_static(SIGNED), + &expected(&unsigned, false) + ) + .expect_err("should refuse a changed media type"), + ResponseError::ContentType + ); + } + + #[test] + fn unchanged_bytes_are_never_accepted_as_signed() { + let unsigned = sha256_hex(UNSIGNED); + assert_eq!( + verify_signed_response( + &embedded_headers(UNSIGNED), + Bytes::from_static(UNSIGNED), + &expected(&unsigned, false), + ) + .expect_err("should refuse an unsigned embedded answer"), + ResponseError::Unsigned + ); + assert_eq!( + verify_signed_response( + &external_headers(UNSIGNED, false), + Bytes::from_static(UNSIGNED), + &expected(&unsigned, true), + ) + .expect_err("should refuse a remote manifest nothing names"), + ResponseError::Unsigned + ); + } + + #[test] + fn a_remote_manifest_needs_a_deliverable_link() { + let unsigned = sha256_hex(UNSIGNED); + assert_eq!( + verify_signed_response( + &external_headers(UNSIGNED, true), + Bytes::from_static(UNSIGNED), + &expected(&unsigned, false), + ) + .expect_err("should refuse a Link the delivery cannot carry"), + ResponseError::LinkUndeliverable + ); + let signed = verify_signed_response( + &external_headers(UNSIGNED, true), + Bytes::from_static(UNSIGNED), + &expected(&unsigned, true), + ) + .expect("should accept a remote manifest the delivery links"); + assert_eq!( + signed.link().as_deref(), + Some("; rel=\"c2pa-manifest\""), + "should name the manifest in a Link" + ); + } + + #[test] + fn repeated_or_malformed_signing_headers_are_refused() { + let unsigned = sha256_hex(UNSIGNED); + let mut repeated = embedded_headers(SIGNED); + repeated.append(INSTANCE_ID_HEADER, HeaderValue::from_static("xmp:iid:0002")); + let mut bad_url = embedded_headers(SIGNED); + bad_url.insert( + VERIFICATION_URL_HEADER, + HeaderValue::from_static("http://verify.example.com/doc-1"), + ); + let mut bad_disposition = embedded_headers(SIGNED); + bad_disposition.insert(DISPOSITION_HEADER, HeaderValue::from_static("sidecar")); + for headers in [repeated, bad_url, bad_disposition] { + assert_eq!( + verify_signed_response( + &headers, + Bytes::from_static(SIGNED), + &expected(&unsigned, false) + ) + .expect_err("should refuse malformed headers"), + ResponseError::Malformed + ); + } + } + + #[test] + fn common_misspellings_name_the_media_type_the_service_lists() { + for (declared, canonical) in [ + ("image/jpg", "image/jpeg"), + ("audio/x-aac", "audio/aac"), + ("Audio/AACP; codecs=aac", "audio/aac"), + ("audio/aac", "audio/aac"), + ("audio/mp4", "audio/mp4"), + ] { + assert_eq!(canonical_media_type(declared), canonical, "{declared}"); + } + } + + #[test] + fn source_references_are_sent_only_when_canonical_and_on_the_claim_host() { + let host = "publisher.example.com"; + assert!( + safe_source_reference("https://publisher.example.com/images/a.png", host), + "should send a canonical https address" + ); + for unsafe_reference in [ + "https://publisher.example.com/images/a.png?token=secret", + "https://user:pass@publisher.example.com/images/a.png", + "https://publisher.example.com/images/a.png#frag", + "http://publisher.example.com/images/a.png", + "https://publisher.example.com:8443/images/a.png", + "https://origin.example.com/images/a.png", + "https://PUBLISHER.example.com/images/a.png", + "https://publisher.example.com/", + ] { + assert!( + !safe_source_reference(unsafe_reference, host), + "should never send {unsafe_reference}" + ); + } + } + + #[test] + fn error_codes_are_read_as_short_identifiers_only() { + let body = serde_json::to_vec(&json!({ + "success": false, + "error": {"code": "TRUSTED_SERVER_POLICY_CHANGED", "message": "stale"}, + "correlation_id": "c-1", + })) + .expect("should serialize"); + assert_eq!( + error_code(&body).as_deref(), + Some("TRUSTED_SERVER_POLICY_CHANGED") + ); + let injected = + serde_json::to_vec(&json!({"error": {"code": "a b\nc"}})).expect("should serialize"); + assert_eq!(error_code(&injected), None, "should drop an unsafe code"); + } +} diff --git a/crates/provenance/encypher/src/prepared_probe.rs b/crates/provenance/encypher/src/prepared_probe.rs index 478b13450f..2521442616 100644 --- a/crates/provenance/encypher/src/prepared_probe.rs +++ b/crates/provenance/encypher/src/prepared_probe.rs @@ -69,7 +69,7 @@ const PRODUCTION_RELEASE_KEY: [u8; 32] = [ /// The lowest probe release this build runs. Raise it when Encypher /// withdraws a release, so a compromised API cannot downgrade the edge to it. -const PRODUCTION_RELEASE_FLOOR: u64 = 1; +const PRODUCTION_RELEASE_FLOOR: u64 = 8; /// A release artifact is about 800 KB; anything over this is not one. const MAX_ARTIFACT_BYTES: usize = 4 * 1024 * 1024; @@ -128,6 +128,7 @@ impl ProbeDescriptor { .unwrap_or_default() } + #[cfg(test)] /// The descriptor as the signing API receives it back. #[must_use] pub(crate) fn to_json(&self) -> Value { @@ -332,7 +333,10 @@ impl fmt::Display for ReleaseError { write!(formatter, "probe release {release} was withdrawn") } Self::MalformedSignature { release } => { - write!(formatter, "probe release {release} carries a malformed signature") + write!( + formatter, + "probe release {release} carries a malformed signature" + ) } Self::Unsigned { release } => write!( formatter, @@ -368,15 +372,21 @@ impl fmt::Display for ReleaseError { formatter, "the probe release reports ABI {reported}, not its descriptor's" ), - Self::PlannerMismatch => formatter - .write_str("the probe release does not report its descriptor's planner"), + Self::PlannerMismatch => { + formatter.write_str("the probe release does not report its descriptor's planner") + } } } } impl std::error::Error for ReleaseError {} -/// Bounds on one call. Exceeding any of them abandons the call. +/// Bounds on one call. +/// +/// Fuel, the total read and the result abandon the call when exceeded. A +/// read over the per-read bound is answered with status 5, and growth past +/// the memory bound fails inside the release, which then answers as it +/// handles an allocation failure. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct ProbeLimits { /// Linear memory one instance may grow to. @@ -392,15 +402,16 @@ pub(crate) struct ProbeLimits { } impl Default for ProbeLimits { - /// The client contract's limits: 1 GiB of memory, 2x10^11 instructions, - /// 64 MiB per read and per result, and at most 1 GiB read per call. + /// Leave room for the host, parsed JSON and the signed module on + /// 128 MiB-class runtimes. Media payload hashing stays outside WASM. + /// Larger structure or result disclosures are refused explicitly. fn default() -> Self { Self { - memory_bytes: 1 << 30, - fuel_per_call: 200_000_000_000, - max_read_bytes: 64 * 1024 * 1024, - max_total_read_bytes: 1 << 30, - max_result_bytes: 64 * 1024 * 1024, + memory_bytes: 32 * 1024 * 1024, + fuel_per_call: 200_000_000, + max_read_bytes: 8 * 1024 * 1024, + max_total_read_bytes: 64 * 1024 * 1024, + max_result_bytes: 4 * 1024 * 1024, } } } @@ -482,7 +493,11 @@ impl fmt::Display for ProbeCallError { match self { Self::Source(error) => write!(formatter, "the probe's source failed a read: {error}"), Self::Refused(refusal) => { - write!(formatter, "the probe refused: {}: {}", refusal.code, refusal.message) + write!( + formatter, + "the probe refused: {}: {}", + refusal.code, refusal.message + ) } Self::Failed(failure) => failure.fmt(formatter), } @@ -541,19 +556,28 @@ impl fmt::Display for ProbeFailure { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Instantiate(error) => { - write!(formatter, "the probe release could not be instantiated: {error}") + write!( + formatter, + "the probe release could not be instantiated: {error}" + ) } Self::ArgumentTooLarge => { formatter.write_str("a probe argument exceeds a 32-bit length") } - Self::Trap(error) => write!(formatter, "the probe release stopped with an error: {error}"), + Self::Trap(error) => write!( + formatter, + "the probe release stopped with an error: {error}" + ), Self::FuelExhausted => formatter.write_str("the probe release ran out of fuel"), Self::ReadBudgetExceeded { budget } => write!( formatter, "the probe release read more than its {budget} byte budget" ), Self::UnknownStatus(status) => { - write!(formatter, "the probe release returned unknown status {status}") + write!( + formatter, + "the probe release returned unknown status {status}" + ) } Self::ResultTooLarge { bytes, limit } => write!( formatter, @@ -563,7 +587,10 @@ impl fmt::Display for ProbeFailure { formatter.write_str("the probe release's result lies outside its memory") } Self::MalformedResult(error) => { - write!(formatter, "the probe release returned a malformed result: {error}") + write!( + formatter, + "the probe release returned a malformed result: {error}" + ) } } } @@ -676,9 +703,9 @@ impl ProbeModule { arg: &Value, source: &S, ) -> Result> { - let mut instance = self - .instantiate(source.size()) - .map_err(|error| ProbeCallError::Failed(ProbeFailure::Instantiate(error.to_string())))?; + let mut instance = self.instantiate(source.size()).map_err(|error| { + ProbeCallError::Failed(ProbeFailure::Instantiate(error.to_string())) + })?; let (op_pointer, op_length) = instance .put(op.as_bytes()) .map_err(ProbeCallError::Failed)?; @@ -774,7 +801,9 @@ impl ProbeModule { ); store.limiter(|state| &mut state.limits); store.set_fuel(self.limits.fuel_per_call)?; - let instance = self.linker.instantiate_and_start(&mut store, &self.module)?; + let instance = self + .linker + .instantiate_and_start(&mut store, &self.module)?; let memory = instance .get_memory(&store, "memory") .ok_or_else(|| wasmi::Error::new("the release exports no memory"))?; @@ -832,6 +861,10 @@ fn refusal(value: &Value) -> Result { } /// A call stopped by the interpreter: out of fuel, or any other trap. +#[allow( + clippy::needless_pass_by_value, + reason = "used directly as a Result error mapper" +)] fn stopped(error: wasmi::Error) -> ProbeCallError { ProbeCallError::Failed(ProbeFailure::stopped(&error)) } @@ -866,6 +899,10 @@ impl wasmi::errors::HostError for ReadRequest {} /// `env.host_read`. Refuses a read over the per-read bound or past the /// source with status 5, and suspends the call for any other. +#[allow( + clippy::needless_pass_by_value, + reason = "wasmi's host import ABI owns its Caller" +)] fn host_read( caller: Caller<'_, HostState>, offset: u64, diff --git a/crates/provenance/encypher/src/prepared_probe/tests.rs b/crates/provenance/encypher/src/prepared_probe/tests.rs index f0edea026e..a9516890d3 100644 --- a/crates/provenance/encypher/src/prepared_probe/tests.rs +++ b/crates/provenance/encypher/src/prepared_probe/tests.rs @@ -17,8 +17,8 @@ use futures::executor::block_on; use serde_json::json; use super::{ - MAX_ARTIFACT_BYTES, PRODUCTION_RELEASE_KEY, ProbeCallError, ProbeDescriptor, ProbeFailure, - ProbeLimits, ProbeModule, ProbeSource, ReleaseAuthority, ReleaseError, + MAX_ARTIFACT_BYTES, ProbeCallError, ProbeDescriptor, ProbeFailure, ProbeLimits, ProbeModule, + ProbeSource, ReleaseAuthority, ReleaseError, }; use crate::media_api::sha256_hex; @@ -34,12 +34,15 @@ const READ_AT: usize = 2048; const STATUS_AT: usize = 3072; const UNKNOWN_AT: usize = 4096; const READ_FAILED_AT: usize = 5120; +/// WebAssembly page size, and the stand-in's initial linear memory. +const PAGE_BYTES: usize = 64 * 1024; +const INITIAL_PAGES: usize = 2; const STAND_IN: &str = r#" (module (import "env" "host_read" (func $host_read (param i64 i32 i32) (result i32))) @EXTRA_IMPORT@ - (memory (export "memory") 2) + (memory (export "memory") @INITIAL_PAGES@) (global $heap (mut i32) (i32.const 16384)) (global $result_ptr (mut i32) (i32.const 0)) (global $result_len (mut i32) (i32.const 0)) @@ -96,9 +99,28 @@ const STAND_IN: &str = r#" (br_if $again (i32.eqz (call $host_read (i64.const 0) (i32.const 4) (i32.const @HEAD_SLOT@))))) (return (call $read_failed)))) - ;; "Grow": 100 pages more memory; reports 0 when refused + ;; "Grow": memory to one page past the bound; reports 0 when refused (if (i32.eq (local.get $kind) (i32.const 71)) - (then (return (call $report (i32.add (i32.const 1) (memory.grow (i32.const 100))))))) + (then (return (call $report + (i32.add (i32.const 1) (memory.grow (i32.const @OVER_PAGES@))))))) + ;; "allocate": memory up to exactly the bound; reports 0 when refused + (if (i32.eq (local.get $kind) (i32.const 97)) + (then (return (call $report + (i32.add (i32.const 1) (memory.grow (i32.const @FIT_PAGES@))))))) + ;; "Beyond": one byte more than the per-read bound + (if (i32.eq (local.get $kind) (i32.const 66)) + (then (return (call $report + (call $host_read (i64.const 0) (i32.const @READ_BOUND_PLUS_ONE@) (i32.const 0)))))) + ;; "bound": exactly the per-read bound, into more than linear memory + (if (i32.eq (local.get $kind) (i32.const 98)) + (then (return (call $report + (call $host_read (i64.const 0) (i32.const @READ_BOUND@) (i32.const 0)))))) + ;; "flood": per-read-bound reads forever, ignoring every status + (if (i32.eq (local.get $kind) (i32.const 102)) + (then + (loop $again + (drop (call $host_read (i64.const 0) (i32.const @READ_BOUND@) (i32.const 0))) + (br $again)))) ;; "echo": the argument as received (if (i32.eq (local.get $kind) (i32.const 101)) (then (return (call $answer (local.get $arg) (local.get $arg_len) (i32.const 0))))) @@ -135,6 +157,9 @@ fn stand_in(abi: u32, planner: &str, extra_import: bool) -> Vec { } else { "" }; + let edge = ProbeLimits::default(); + let fit_pages = edge.memory_bytes / PAGE_BYTES - INITIAL_PAGES; + let read_bound = edge.max_read_bytes; let text = STAND_IN .replace("@EXTRA_IMPORT@", extra) .replace("@ABI@", &abi.to_string()) @@ -155,7 +180,12 @@ fn stand_in(abi: u32, planner: &str, extra_import: bool) -> Vec { .replace("@UNKNOWN_LEN@", &UNKNOWN_JSON.len().to_string()) .replace("@READ_FAILED_AT@", &READ_FAILED_AT.to_string()) .replace("@READ_FAILED_JSON@", &wat_string(READ_FAILED_JSON)) - .replace("@READ_FAILED_LEN@", &READ_FAILED_JSON.len().to_string()); + .replace("@READ_FAILED_LEN@", &READ_FAILED_JSON.len().to_string()) + .replace("@INITIAL_PAGES@", &INITIAL_PAGES.to_string()) + .replace("@FIT_PAGES@", &fit_pages.to_string()) + .replace("@OVER_PAGES@", &(fit_pages + 1).to_string()) + .replace("@READ_BOUND@", &read_bound.to_string()) + .replace("@READ_BOUND_PLUS_ONE@", &(read_bound + 1).to_string()); wat::parse_str(&text).expect("should assemble the stand-in release") } @@ -163,7 +193,13 @@ fn release_key() -> SigningKey { SigningKey::from_bytes(&[7; 32]) } -fn signed(key: &SigningKey, abi: u64, release: u64, planner: &str, sha256: &str) -> ProbeDescriptor { +fn signed( + key: &SigningKey, + abi: u64, + release: u64, + planner: &str, + sha256: &str, +) -> ProbeDescriptor { let mut descriptor = ProbeDescriptor { abi, planner_id: planner.to_owned(), @@ -184,8 +220,13 @@ fn authority(floor: u64, revoked: Vec) -> ReleaseAuthority { fn loaded(limits: ProbeLimits) -> ProbeModule { let artifact = stand_in(1, &planner(), false); let descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&artifact)); - block_on(ProbeModule::load(&authority(1, Vec::new()), &descriptor, &artifact, limits)) - .expect("should load the signed stand-in") + block_on(ProbeModule::load( + &authority(1, Vec::new()), + &descriptor, + &artifact, + limits, + )) + .expect("should load the signed stand-in") } fn load_error(artifact: &[u8], descriptor: &ProbeDescriptor) -> ReleaseError { @@ -231,6 +272,7 @@ impl Future for YieldOnce { /// An in-memory source whose every read first suspends. struct AsyncSource { bytes: Vec, + size: u64, fail: bool, reads: RefCell>, suspensions: Cell, @@ -240,6 +282,7 @@ impl AsyncSource { fn new(bytes: &[u8]) -> Self { Self { bytes: bytes.to_vec(), + size: bytes.len() as u64, fail: false, reads: RefCell::new(Vec::new()), suspensions: Cell::new(0), @@ -253,6 +296,15 @@ impl AsyncSource { } } + /// A source that claims `size` bytes and holds none, for reads the host + /// must refuse before asking it. + fn claiming(size: u64) -> Self { + Self { + size, + ..Self::new(&[]) + } + } + fn reads(&self) -> Vec<(u64, usize)> { self.reads.borrow().clone() } @@ -262,10 +314,14 @@ impl ProbeSource for AsyncSource { type Error = VersionChanged; fn size(&self) -> u64 { - self.bytes.len() as u64 + self.size } - async fn read_exact_at(&self, offset: u64, destination: &mut [u8]) -> Result<(), VersionChanged> { + async fn read_exact_at( + &self, + offset: u64, + destination: &mut [u8], + ) -> Result<(), VersionChanged> { self.suspensions.set(self.suspensions.get() + 1); YieldOnce::default().await; self.reads.borrow_mut().push((offset, destination.len())); @@ -278,29 +334,57 @@ impl ProbeSource for AsyncSource { } } +/// Release 8 as the production signing service advertised it, signed by +/// Encypher's production release key. Only the public descriptor is kept; +/// its artifact is not part of this repository. +const PRODUCTION_RELEASE_8: &str = r#"{ + "abi": 1, + "planner_id": "0685f74f2b47b171a62be7d8f38224c85b5fdc1fc849d316e8c93910406ece04", + "release": 8, + "sha256": "45c8e965b33e6763a52a9164392fb584f1dd6d993e9583a9f51945caa4ebed59", + "signature": "Mge6x1crHLQtuvn0eDjtapacOVTiEfU7/GOoxRd09e2bFGZaiSn9CvBlf/hSZs6Auvb2aRhWDVeOqnw+iTMMCw==" +}"#; + #[test] -fn production_authority_pins_the_published_release_key() { - let hex: String = PRODUCTION_RELEASE_KEY - .iter() - .map(|byte| format!("{byte:02x}")) - .collect(); +fn production_authority_accepts_only_releases_encypher_signed() { + let authority = ReleaseAuthority::production(); + let published: serde_json::Value = + serde_json::from_str(PRODUCTION_RELEASE_8).expect("should parse the fixture"); + let published = ProbeDescriptor::parse(&published).expect("should be a descriptor"); assert_eq!( - hex, "9ae0bf4aea5140008d2f48e6c08691360a7fe23e0d36f24522c770b7646b75d9", - "should pin the key the client contract publishes" + authority.select(std::slice::from_ref(&published)), + Ok(&published), + "should accept the release Encypher published" ); - let authority = ReleaseAuthority::production(); - let descriptor = signed(&release_key(), 1, 3, &planner(), &"b".repeat(64)); + + let mut retargeted = published.clone(); + retargeted.sha256 = "b".repeat(64); assert_eq!( - authority.verify(&descriptor), - Err(ReleaseError::Unsigned { release: 3 }), - "production should refuse a release signed by any other key" + authority.verify(&retargeted), + Err(ReleaseError::Unsigned { release: 8 }), + "should refuse the published signature over another artifact" + ); + let foreign = signed( + &release_key(), + 1, + 8, + &published.planner_id, + &published.sha256, + ); + assert_eq!( + authority.verify(&foreign), + Err(ReleaseError::Unsigned { release: 8 }), + "should refuse the same release signed by any other key" ); } #[test] fn descriptors_parse_only_in_the_contract_shape() { let descriptor = signed(&release_key(), 1, 3, &planner(), &"b".repeat(64)); - assert_eq!(ProbeDescriptor::parse(&descriptor.to_json()), Some(descriptor.clone())); + assert_eq!( + ProbeDescriptor::parse(&descriptor.to_json()), + Some(descriptor.clone()) + ); assert_eq!( descriptor.canonical_json(), format!( @@ -324,7 +408,11 @@ fn descriptors_parse_only_in_the_contract_shape() { let mut textual = descriptor.to_json(); textual["release"] = json!("3"); for malformed in [&upper, &short, &unsigned, &textual] { - assert_eq!(ProbeDescriptor::parse(malformed), None, "should refuse {malformed}"); + assert_eq!( + ProbeDescriptor::parse(malformed), + None, + "should refuse {malformed}" + ); } let list = json!([descriptor.to_json(), upper, "noise"]); assert_eq!(ProbeDescriptor::parse_list(Some(&list)), vec![descriptor]); @@ -339,10 +427,22 @@ fn verification_refuses_every_tampered_descriptor() { let mut retargeted = good.clone(); retargeted.sha256 = "d".repeat(64); - assert_eq!(authority.verify(&retargeted), Err(ReleaseError::Unsigned { release: 3 })); + assert_eq!( + authority.verify(&retargeted), + Err(ReleaseError::Unsigned { release: 3 }) + ); - let foreign = signed(&SigningKey::from_bytes(&[9; 32]), 1, 3, &planner(), &"b".repeat(64)); - assert_eq!(authority.verify(&foreign), Err(ReleaseError::Unsigned { release: 3 })); + let foreign = signed( + &SigningKey::from_bytes(&[9; 32]), + 1, + 3, + &planner(), + &"b".repeat(64), + ); + assert_eq!( + authority.verify(&foreign), + Err(ReleaseError::Unsigned { release: 3 }) + ); for signature in ["not base64!", "AAAA", &STANDARD.encode([0_u8; 63])] { let mut garbled = good.clone(); @@ -355,7 +455,10 @@ fn verification_refuses_every_tampered_descriptor() { } let future_abi = signed(&release_key(), 2, 3, &planner(), &"b".repeat(64)); - assert_eq!(authority.verify(&future_abi), Err(ReleaseError::AbiUnsupported { abi: 2 })); + assert_eq!( + authority.verify(&future_abi), + Err(ReleaseError::AbiUnsupported { abi: 2 }) + ); assert_eq!( authority.verify(&future_abi).map_err(|error| error.code()), Err("PROBE_UPGRADE_REQUIRED") @@ -364,11 +467,17 @@ fn verification_refuses_every_tampered_descriptor() { let old = signed(&release_key(), 1, 1, &planner(), &"b".repeat(64)); assert_eq!( authority.verify(&old), - Err(ReleaseError::BelowFloor { release: 1, floor: 2 }) + Err(ReleaseError::BelowFloor { + release: 1, + floor: 2 + }) ); let withdrawn = signed(&release_key(), 1, 4, &planner(), &"C".repeat(64)); - assert_eq!(authority.verify(&withdrawn), Err(ReleaseError::Revoked { release: 4 })); + assert_eq!( + authority.verify(&withdrawn), + Err(ReleaseError::Revoked { release: 4 }) + ); } #[test] @@ -403,7 +512,10 @@ fn loading_refuses_artifacts_that_are_not_the_signed_release() { let mut tampered = artifact.clone(); let last = tampered.len() - 1; tampered[last] ^= 1; - assert_eq!(load_error(&tampered, &descriptor), ReleaseError::DigestMismatch); + assert_eq!( + load_error(&tampered, &descriptor), + ReleaseError::DigestMismatch + ); let oversized = vec![0_u8; MAX_ARTIFACT_BYTES + 1]; let oversized_descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&oversized)); @@ -433,7 +545,10 @@ fn loading_refuses_artifacts_that_are_not_the_signed_release() { fn loading_refuses_a_signed_release_that_breaks_the_abi() { let importing = stand_in(1, &planner(), true); let descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&importing)); - assert_eq!(load_error(&importing, &descriptor), ReleaseError::ImportsMismatch); + assert_eq!( + load_error(&importing, &descriptor), + ReleaseError::ImportsMismatch + ); let misreporting = stand_in(2, &planner(), false); let descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&misreporting)); @@ -443,7 +558,13 @@ fn loading_refuses_a_signed_release_that_breaks_the_abi() { ); let other_planner = stand_in(1, &"f".repeat(64), false); - let descriptor = signed(&release_key(), 1, 3, &planner(), &sha256_hex(&other_planner)); + let descriptor = signed( + &release_key(), + 1, + 3, + &planner(), + &sha256_hex(&other_planner), + ); assert_eq!( load_error(&other_planner, &descriptor), ReleaseError::PlannerMismatch @@ -464,10 +585,17 @@ fn reads_are_awaited_in_order_and_the_call_resumes_without_replay() { vec![(0, 4), (4, 4)], "each read should reach the source once, in order" ); - assert_eq!(source.suspensions.get(), 2, "each read should suspend the call"); + assert_eq!( + source.suspensions.get(), + 2, + "each read should suspend the call" + ); let again = block_on(probe.call("read", &json!({}), &source)).expect("should read again"); - assert_eq!(again, value, "a second call should start from a fresh instance"); + assert_eq!( + again, value, + "a second call should start from a fresh instance" + ); assert_eq!(source.reads().len(), 4); } @@ -476,8 +604,8 @@ fn the_argument_reaches_the_release_intact() { let probe = loaded(ProbeLimits::default()); let arg = json!({"plan": {"items": [1, 2, 3]}, "text": "caf\u{e9} \"quoted\""}); - let echoed = block_on(probe.call_without_source::("echo", &arg)) - .expect("should echo"); + let echoed = + block_on(probe.call_without_source::("echo", &arg)).expect("should echo"); assert_eq!(echoed, arg); } @@ -493,28 +621,55 @@ fn reads_outside_the_source_or_memory_are_refused_without_asking_the_source() { let memory = block_on(probe.call("memory", &json!({}), &source)).expect("should answer"); assert_eq!(memory, json!({"status": 4})); - assert!(source.reads().is_empty(), "no refused read should reach the source"); + assert!( + source.reads().is_empty(), + "no refused read should reach the source" + ); } #[test] -fn a_read_over_the_per_read_bound_is_refused_with_status_5() { - let probe = loaded(ProbeLimits { - max_read_bytes: 3, - ..ProbeLimits::default() - }); - let source = AsyncSource::new(b"abcdefgh"); +fn the_edge_per_read_bound_admits_its_size_and_refuses_one_byte_more() { + let probe = loaded(ProbeLimits::default()); + let bound = u64::from(ProbeLimits::default().max_read_bytes); + let source = AsyncSource::claiming(bound + 1); + + let beyond = block_on(probe.call("Beyond", &json!({}), &source)).expect("should answer"); + assert_eq!( + beyond, + json!({"status": 5}), + "a read over the bound should be refused" + ); + + let at_bound = block_on(probe.call("bound", &json!({}), &source)).expect("should answer"); + assert_eq!( + at_bound, + json!({"status": 4}), + "a read at the bound should pass the bound and stop only at linear memory" + ); + assert!( + source.reads().is_empty(), + "neither read should reach the source" + ); +} + +#[test] +fn the_edge_total_read_budget_stops_a_release_that_keeps_reading() { + let probe = loaded(ProbeLimits::default()); + let source = AsyncSource::claiming(u64::from(ProbeLimits::default().max_read_bytes)); - let error = block_on(probe.call("read", &json!({}), &source)).expect_err("should refuse"); + let error = block_on(probe.call("flood", &json!({}), &source)).expect_err("should stop"); assert!( - matches!(&error, ProbeCallError::Refused(refusal) if refusal.code == "READ_FAILED"), - "the release should see the refusal: {error}" + matches!( + error, + ProbeCallError::Failed(ProbeFailure::ReadBudgetExceeded { .. }) + ), + "the read budget, not fuel, should stop the release: {error}" ); - assert!(source.reads().is_empty()); } #[test] -fn the_total_read_budget_abandons_the_call() { +fn the_read_budget_counts_every_admitted_read() { let probe = loaded(ProbeLimits { max_total_read_bytes: 8, ..ProbeLimits::default() @@ -538,34 +693,52 @@ fn a_failed_read_abandons_the_call_with_the_source_error() { let error = block_on(probe.call("read", &json!({}), &source)).expect_err("should fail"); assert!(matches!(error, ProbeCallError::Source(VersionChanged))); - assert_eq!(source.reads(), vec![(0, 4)], "no read should follow the failure"); + assert_eq!( + source.reads(), + vec![(0, 4)], + "no read should follow the failure" + ); } #[test] -fn fuel_memory_and_result_bounds_stop_the_release() { - let fuel_bound = loaded(ProbeLimits { - fuel_per_call: 1_000_000, - ..ProbeLimits::default() - }); - let spin = block_on(fuel_bound.call_without_source::("spin", &json!({}))); +fn edge_fuel_stops_a_release_that_never_returns() { + let probe = loaded(ProbeLimits::default()); + + let spin = block_on(probe.call_without_source::("spin", &json!({}))); + assert!(matches!( spin, Err(ProbeCallError::Failed(ProbeFailure::FuelExhausted)) )); +} - let memory_bound = loaded(ProbeLimits { - memory_bytes: 1024 * 1024, - ..ProbeLimits::default() - }); - let refused = block_on(memory_bound.call_without_source::("Grow", &json!({}))) +#[test] +fn edge_memory_grows_to_its_bound_and_no_further() { + let probe = loaded(ProbeLimits::default()); + + let to_bound = block_on(probe.call_without_source::("allocate", &json!({}))) .expect("should answer"); - assert_eq!(refused, json!({"status": 0}), "growth past the bound should fail"); - let unbounded = loaded(ProbeLimits::default()); - let grown = block_on(unbounded.call_without_source::("Grow", &json!({}))) + assert_eq!( + to_bound, + json!({"status": 3}), + "growth to exactly the bound should succeed" + ); + + let past_bound = block_on(probe.call_without_source::("Grow", &json!({}))) .expect("should answer"); - assert_eq!(grown, json!({"status": 3}), "growth within the bound should succeed"); + assert_eq!( + past_bound, + json!({"status": 0}), + "growth one page past the bound should fail inside the release" + ); +} + +#[test] +fn an_oversized_result_is_refused() { + let probe = loaded(ProbeLimits::default()); + + let huge = block_on(probe.call_without_source::("huge", &json!({}))); - let huge = block_on(unbounded.call_without_source::("huge", &json!({}))); assert!(matches!( huge, Err(ProbeCallError::Failed(ProbeFailure::ResultTooLarge { .. })) @@ -577,7 +750,10 @@ fn traps_and_error_objects_are_reported_as_such() { let probe = loaded(ProbeLimits::default()); let trap = block_on(probe.call_without_source::("x", &json!({}))); - assert!(matches!(trap, Err(ProbeCallError::Failed(ProbeFailure::Trap(_))))); + assert!(matches!( + trap, + Err(ProbeCallError::Failed(ProbeFailure::Trap(_))) + )); let unknown = block_on(probe.call_without_source::("unknown", &json!({}))) .expect_err("should refuse"); diff --git a/crates/provenance/encypher/src/processor.rs b/crates/provenance/encypher/src/processor.rs index 57af424567..3604e0388f 100644 --- a/crates/provenance/encypher/src/processor.rs +++ b/crates/provenance/encypher/src/processor.rs @@ -1,19 +1,24 @@ -//! Per-document signing flow: gates, hold, KV cache, sign call, outcomes, logging. +//! Per-document signing flow: gates, hold, KV cache, sign calls, outcomes, logging. //! //! A document moves through four phases: //! //! 1. **Admit** (the middleware's preparation, with no I/O): the request //! gates decide whether the document is held at all. //! 2. **Extract** (synchronous): at end of input the held document's article -//! boundary, visible text and text hash are computed. +//! boundary, visible text and text hash are computed, and its embedded +//! media found. //! 3. **Resolve** (the asynchronous finish): policy and plan caches, the policy -//! and sign calls, and cache writes. On runtimes with a timer every call is -//! raced against `max_hold_ms`, keeping a reserve for the skip write. +//! and sign calls, and cache writes, then each embedded media item signed +//! through [`media`]. On runtimes with a timer every call is raced against +//! one `max_hold_ms` budget, keeping a reserve for the skip write. //! 4. **Render** (synchronous): the plan's markers and the manifest meta tag are -//! applied to the held document. +//! applied to the held document, then each signed media item replaces its +//! exact bytes. //! -//! Every module failure emits the held document unchanged. +//! Every module failure emits the held document unchanged, or, for one +//! media item, leaves that item as written and reports it unsigned. +use std::borrow::Cow; use std::io; use std::pin::pin; use std::sync::Arc; @@ -29,6 +34,7 @@ use serde::{Deserialize, Serialize}; use crate::api::{self, ApiError, EmbeddingPlan, SignRequest}; use crate::boundary::{self, Boundary}; +use crate::embedded; use crate::{EncypherConfig, INTEGRATION_ID, embed, text}; use trusted_server_core::integrations::ensure_integration_backend_with_timeout; use trusted_server_core::platform::{ @@ -36,6 +42,11 @@ use trusted_server_core::platform::{ }; use trusted_server_core::streaming_processor::{AsyncFinish, StreamProcessor}; +mod media; +mod prepared; + +pub(crate) use media::EncypherMedia; + const SIGN_PATH: &str = "/api/v1/public/cdn/sign"; const POLICY_PATH: &str = "/api/v1/public/cdn/trusted-server/policy"; const WORKER_TOKEN_HEADER: &str = "x-encypher-worker-token"; @@ -63,7 +74,8 @@ const SKIP_PATH_PREFIXES: &[&str] = &[ "/sitemap", "/.well-known/", ]; -/// Asset suffixes never signed (ported from the Fastly worker's `shouldSkipPath`). +/// Asset suffixes the article path never signs (ported from the Fastly +/// worker's `shouldSkipPath`). Media at these paths is signed by [`media`]. const SKIP_PATH_SUFFIXES: &[&str] = &[ ".xml", ".json", ".rss", ".atom", ".css", ".js", ".woff", ".woff2", ".png", ".jpg", ".gif", ".svg", ".ico", @@ -103,19 +115,24 @@ pub(crate) struct PageRequest { /// The reader's request method. pub(crate) method: Method, /// The scheme, host and path the reader asked for, without the query, - /// which is the address the article is signed for. + /// which is the address the article is signed for. On a host that + /// serves several publishers by path it includes the publisher's prefix. pub(crate) page_url: String, + /// The path prefix request addressing removed, such as + /// `/www.example.com`, or empty. The path gates read the publisher's own + /// path, without it. + pub(crate) removed_prefix: String, /// The status of the response the document is the body of. pub(crate) response_status: u16, /// Whether the origin marked its response as one reader's. pub(crate) origin_response_is_private: bool, } -/// Reason code for a held document's outcome. Sign failures are also stored -/// as the reason of a negative (`skip`) cache entry. +/// Reason code for a held document's or a media item's outcome. Sign +/// failures are also stored as the reason of a negative (`skip`) cache entry. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] -enum Reason { +pub(crate) enum Reason { Signed, CacheHit, DocumentTooLarge, @@ -148,10 +165,62 @@ enum Reason { NoPlan, InvalidPlan, AlignmentFailed, + /// The media is over `max_media_bytes`. + MediaTooLarge, + /// The media is delivered outside the claimed domain. + OutsideClaim, + /// Core named no single hostname the source was fetched from. + InvalidOriginHost, + /// The captured bytes are not the bytes core hashed. + SourceDigestMismatch, + /// A `data:` URI's payload does not decode. + MalformedDataUri, + /// The endpoint answered `409` to a reused replay key. + IdempotencyConflict, + /// The endpoint no longer holds the result for the replay key. + ResultPurged, + /// The endpoint refused the request body's size. + PayloadTooLarge, + /// The signed bytes are of another media type than the source. + SignedTypeMismatch, + /// The signed bytes are not the bytes the answer names. + SignedDigestMismatch, + /// A signing header is missing, repeated or malformed. + InvalidSignedResponse, + /// The manifest is remote and needs a `Link` the delivery cannot carry. + ManifestUndeliverable, + /// The answer carries no manifest and no reference to one. + UnsignedResponse, + /// The response's policies are not visible to the module, so an inline + /// style edit could break a style hash. + CspUnknown, + /// A page policy admits the inline style by hash. + CspStyleHash, + /// The element carries a subresource integrity value. + IntegrityAttribute, + /// The URI is written with an HTML character reference. + CharacterReference, + /// The URI is written with a CSS escape. + CssEscape, + /// No standalone SVG document is proven to parse to the tree the browser + /// built from the inline SVG. + InlineSvgNotEquivalent, + /// The inline SVG carries provenance of its own and is not, as written, + /// a standalone SVG equivalent to the browser's tree, so it is never + /// rewritten. + SvgProvenanceNotEquivalent, + /// Original SVG provenance protects the entire range from inline edits + /// or replacement with a new, unchained signature. + SvgExistingProvenance, + /// The signer changed the inline SVG beyond adding its manifest. + InlineSvgAltered, + /// The document's media made its most endpoint calls; the item is signed + /// on a later view. + Deferred, } impl Reason { - fn code(self) -> &'static str { + pub(crate) fn code(self) -> &'static str { match self { Self::Signed => "signed", Self::CacheHit => "cache_hit", @@ -185,6 +254,29 @@ impl Reason { Self::NoPlan => "no_plan", Self::InvalidPlan => "invalid_plan", Self::AlignmentFailed => "alignment_failed", + Self::MediaTooLarge => "media_too_large", + Self::OutsideClaim => "outside_claim", + Self::InvalidOriginHost => "invalid_origin_host", + Self::SourceDigestMismatch => "source_digest_mismatch", + Self::MalformedDataUri => "malformed_data_uri", + Self::IdempotencyConflict => "idempotency_conflict", + Self::ResultPurged => "result_purged", + Self::PayloadTooLarge => "payload_too_large", + Self::SignedTypeMismatch => "signed_type_mismatch", + Self::SignedDigestMismatch => "signed_digest_mismatch", + Self::InvalidSignedResponse => "invalid_signed_response", + Self::ManifestUndeliverable => "manifest_undeliverable", + Self::UnsignedResponse => "unsigned_response", + Self::CspUnknown => "csp_unknown", + Self::CspStyleHash => "csp_style_hash", + Self::IntegrityAttribute => "integrity_attribute", + Self::CharacterReference => "character_reference", + Self::CssEscape => "css_escape", + Self::InlineSvgNotEquivalent => "inline_svg_not_equivalent", + Self::SvgProvenanceNotEquivalent => "svg_provenance_not_equivalent", + Self::SvgExistingProvenance => "svg_existing_provenance", + Self::InlineSvgAltered => "inline_svg_altered", + Self::Deferred => "deferred", } } } @@ -292,6 +384,7 @@ pub(crate) struct Signer { config: EncypherConfig, max_document_bytes: usize, sign_url: String, + media_sign_url: String, policy_url: String, policy_key: String, policy_skip_key: String, @@ -308,6 +401,11 @@ impl Signer { /// document. pub(crate) fn new(config: EncypherConfig, max_document_bytes: usize) -> Self { let sign_url = format!("{}{SIGN_PATH}", config.api_base_url.trim_end_matches('/')); + let media_sign_url = format!( + "{}{}", + config.api_base_url.trim_end_matches('/'), + crate::media_api::MEDIA_SIGN_PATH + ); let policy_url = format!("{}{POLICY_PATH}", config.api_base_url.trim_end_matches('/')); let context_hash = text::content_hash_hex(&format!( "{}\n{}\n{}", @@ -323,6 +421,7 @@ impl Signer { config, max_document_bytes, sign_url, + media_sign_url, policy_url, policy_key: format!("{POLICY_KEY_PREFIX}:{cache_context}"), policy_skip_key: format!("{POLICY_SKIP_KEY_PREFIX}:{cache_context}"), @@ -332,16 +431,58 @@ impl Signer { } } -/// Holds the document of an admitted page until it ends. -pub(crate) fn document_processor(signer: &Arc, request: PageRequest) -> DocumentProcessor { +/// What a held document is, which decides what in it is signed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Surface { + /// An HTML page: its article's text, then its embedded media. + Page, + /// A stylesheet: the media its `url()`s embed. + Stylesheet, +} + +/// Holds the document of an admitted page until it ends. `policies` are the +/// response's `Content-Security-Policy` values, `None` when core cannot +/// show them, which decide whether inline style media may be edited. +pub(crate) fn document_processor( + signer: &Arc, + request: PageRequest, + policies: Option>, +) -> DocumentProcessor { + held_processor(signer, request, Surface::Page, policies) +} + +/// Holds an admitted stylesheet until it ends. +pub(crate) fn stylesheet_processor( + signer: &Arc, + request: PageRequest, +) -> DocumentProcessor { + held_processor(signer, request, Surface::Stylesheet, None) +} + +fn held_processor( + signer: &Arc, + request: PageRequest, + surface: Surface, + policies: Option>, +) -> DocumentProcessor { DocumentProcessor { signer: Arc::clone(signer), request, + surface, + policies, held: Vec::new(), state: DocumentState::Holding, } } +/// What a held document's signing works for. +struct Target<'a> { + request: &'a PageRequest, + surface: Surface, + /// The response's policy values, `None` when core cannot show them. + policies: Option<&'a [String]>, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum DocumentState { /// Accumulating the document. @@ -358,6 +499,8 @@ enum DocumentState { pub(crate) struct DocumentProcessor { signer: Arc, request: PageRequest, + surface: Surface, + policies: Option>, held: Vec, state: DocumentState, } @@ -396,31 +539,40 @@ impl StreamProcessor for DocumentProcessor { if self.state != DocumentState::Complete { return None; } - let extracted = std::str::from_utf8(held) - .map_err(|_| Reason::NotUtf8) - .and_then(|html| Ok((html, self.signer.extract(html)?))); - let (html, article) = match extracted { - Ok(extracted) => extracted, - Err(reason) => { + let Ok(document) = std::str::from_utf8(held) else { + log_outcome(false, Reason::NotUtf8); + return None; + }; + let (article, media) = match self.surface { + Surface::Page => ( + Some(self.signer.extract(document)), + embedded::scan_html(document, self.policies.as_deref()), + ), + Surface::Stylesheet => (None, embedded::scan_stylesheet(document)), + }; + let article = match article { + Some(Ok(article)) => Some(article), + Some(Err(reason)) => { log_outcome(false, reason); - return None; + None } + None => None, }; + if article.is_none() && media.is_empty() { + return None; + } let signer = &*self.signer; - let request = &self.request; + let target = Target { + request: &self.request, + surface: self.surface, + policies: self.policies.as_deref(), + }; let deadline = Duration::from_millis(signer.config.max_hold_ms).saturating_add(BACKSTOP_MARGIN); Some(AsyncFinish::new(deadline, async move { - match signer.resolve(request, services, html, &article).await { - Ok((signed, reason)) => { - log_outcome(true, reason); - signed.into_bytes() - } - Err(reason) => { - log_outcome(false, reason); - held.to_vec() - } - } + signer + .resolve_document(&target, services, document, article, media) + .await })) } } @@ -497,21 +649,38 @@ fn log_outcome(signed: bool, reason: Reason) { ); } -/// Whether `path` is one this module never signs: an API, feed, admin or -/// well-known path, or an asset. +/// Whether `path` is one the article path never signs: an API, feed, admin +/// or well-known path, or an asset. pub(crate) fn is_excluded_path(path: &str) -> bool { let lower = path.to_ascii_lowercase(); - SKIP_PATH_PREFIXES - .iter() - .any(|prefix| lower.starts_with(prefix)) + is_excluded_prefix(&lower) || SKIP_PATH_SUFFIXES .iter() .any(|suffix| lower.ends_with(suffix)) } +/// Whether `path` is an API, feed, admin or well-known path, where no media +/// is signed either. Asset suffixes are not excluded: media is. +pub(crate) fn is_excluded_prefix(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + SKIP_PATH_PREFIXES + .iter() + .any(|prefix| lower.starts_with(prefix)) +} + impl Signer { - /// Apply the request gates. Only an admitted document is held. + /// Apply the request gates to a page. Only an admitted document is held. pub(crate) fn admit(&self, request: PageRequest) -> Result { + self.gate(request, true) + } + + /// Apply the request gates to a stylesheet, whose `.css` path the page + /// gates exclude. + pub(crate) fn admit_stylesheet(&self, request: PageRequest) -> Result { + self.gate(request, false) + } + + fn gate(&self, request: PageRequest, page: bool) -> Result { if request.method != Method::GET { return Err(Gate::Method); } @@ -523,15 +692,31 @@ impl Signer { } let page_url = url::Url::parse(&request.page_url).map_err(|_| Gate::InvalidPageUrl)?; let host = page_url.host_str().ok_or(Gate::InvalidPageUrl)?; - if host.strip_prefix("www.").unwrap_or(host) != self.config.domain { + if !self.claims_host(host) { return Err(Gate::HostMismatch); } - if is_excluded_path(page_url.path()) { + // The publisher's own path, as its site names it. + let path = page_url.path(); + let path = path + .strip_prefix(request.removed_prefix.as_str()) + .filter(|rest| rest.starts_with('/')) + .unwrap_or(path); + let excluded = if page { + is_excluded_path(path) + } else { + is_excluded_prefix(path) + }; + if excluded { return Err(Gate::ExcludedPath); } Ok(request) } + /// Whether `host` is the claimed domain or its `www.` name. + pub(crate) fn claims_host(&self, host: &str) -> bool { + host.strip_prefix("www.").unwrap_or(host) == self.config.domain + } + /// Find the article and its signable text, with no I/O. fn extract(&self, html: &str) -> Result { let boundary = boundary::find_article_boundary(html, Some(&self.config.article_selector)) @@ -569,22 +754,105 @@ impl Signer { Deadline::new(services.timer(), SKIP_WRITE_RESERVE) } - /// Resolve the plan for `article` and render it into `html`. - async fn resolve( + /// Sign what a held document carries: the article's text first, as the + /// page path always has, then each embedded media item, within one + /// `max_hold_ms` budget. Returns the bytes to emit. + async fn resolve_document( &self, - request: &PageRequest, + target: &Target<'_>, services: &RuntimeServices, - html: &str, - article: &Article, - ) -> Result<(String, Reason), Reason> { - let kv = services.kv_store(); + document: &str, + article: Option
, + media: embedded::Scan, + ) -> Vec { + let request = target.request; let mut work = Deadline::new( services.timer(), Duration::from_millis(self.config.max_hold_ms).saturating_sub(SKIP_WRITE_RESERVE), ); work.start().await; + let policy_digest = match self + .policy_digest(&request.page_url, services, services.kv_store(), &mut work) + .await + { + Ok(policy_digest) => policy_digest, + Err(reason) => { + if article.is_some() { + log_outcome(false, reason); + } + media::log_unsigned(&media, reason); + return document.as_bytes().to_vec(); + } + }; + let mut output = Cow::Borrowed(document); + if let Some(article) = &article { + match self + .resolve( + request, + services, + document, + article, + &policy_digest, + &mut work, + ) + .await + { + Ok((signed, reason)) => { + log_outcome(true, reason); + output = Cow::Owned(signed); + } + Err(reason) => log_outcome(false, reason), + } + } + if media.is_empty() { + return output.into_owned().into_bytes(); + } + // Markers go into text and the manifest meta tag into the head, never + // into an attribute, a style or an SVG, so a rescan finds the same + // media at its new offsets. + let media = match (&output, target.surface) { + (Cow::Owned(signed), Surface::Page) => embedded::scan_html(signed, target.policies), + _ => media, + }; + // The text plan may come from the cached policy, but media is signed + // or reused only under the policy the service answers now, so a + // revoked credential cannot reuse a cached media result. + let media_policy = match self + .request_policy(&request.page_url, services, &mut work) + .await + { + Ok(policy_digest) => policy_digest, + Err(reason) => { + media::log_unsigned(&media, reason); + return output.into_owned().into_bytes(); + } + }; + let embedded = media::EmbeddedJob { + request, + surface: target.surface, + policy_digest: &media_policy, + }; + match self + .sign_embedded(services, &embedded, &output, &media, &mut work) + .await + { + Some(signed) => signed.into_bytes(), + None => output.into_owned().into_bytes(), + } + } - let policy_digest = self.policy_digest(request, services, kv, &mut work).await?; + /// Resolve the plan for `article` under `policy_digest` and render it + /// into `html`. + async fn resolve( + &self, + request: &PageRequest, + services: &RuntimeServices, + html: &str, + article: &Article, + policy_digest: &str, + work: &mut Deadline<'_>, + ) -> Result<(String, Reason), Reason> { + let kv = services.kv_store(); let suffix = format!( "{}:{policy_digest}:{}", self.cache_context, @@ -592,12 +860,12 @@ impl Signer { ); let plan_key = format!("{PLAN_KEY_PREFIX}:{suffix}"); let skip_key = format!("{PLAN_SKIP_KEY_PREFIX}:{suffix}"); - let outcome = match self.read::(kv, &plan_key, &mut work).await { + let outcome = match self.read::(kv, &plan_key, work).await { Ok(Some(CacheEntry::Plan { plan, manifest_url, .. })) => { if api::validate_plan(&plan, manifest_url.as_deref(), article.scalars).is_err() { - self.delete_invalid_entry(kv, &plan_key, &mut work).await; + self.delete_invalid_entry(kv, &plan_key, work).await; return Err(Reason::CachedPlanInvalid); } return render( @@ -611,21 +879,21 @@ impl Signer { .map_err(render_reason); } Ok(Some(CacheEntry::Skip { .. })) => { - self.delete_invalid_entry(kv, &plan_key, &mut work).await; + self.delete_invalid_entry(kv, &plan_key, work).await; return Err(Reason::CachedEntryInvalid); } Err(Reason::Timeout) => Err(Reason::Timeout), Err(reason) => return Err(reason), - Ok(None) => match self.read::(kv, &skip_key, &mut work).await { + Ok(None) => match self.read::(kv, &skip_key, work).await { Ok(Some(CacheEntry::Skip { .. })) => return Err(Reason::CachedSkip), Ok(Some(CacheEntry::Plan { .. })) => { - self.delete_invalid_entry(kv, &skip_key, &mut work).await; + self.delete_invalid_entry(kv, &skip_key, work).await; return Err(Reason::CachedEntryInvalid); } Err(Reason::Timeout) => Err(Reason::Timeout), Err(reason) => return Err(reason), Ok(None) => self - .request_plan(request, services, html, article, &policy_digest, &mut work) + .request_plan(request, services, html, article, policy_digest, work) .await .and_then(|(plan, manifest_url)| { api::validate_plan(&plan, manifest_url.as_deref(), article.scalars) @@ -651,14 +919,8 @@ impl Signer { expires_at: Self::expires_at(self.config.plan_ttl_seconds), ttl_less: false, }; - self.write( - kv, - &plan_key, - entry, - self.config.plan_ttl_seconds, - &mut work, - ) - .await; + self.write(kv, &plan_key, entry, self.config.plan_ttl_seconds, work) + .await; Ok((signed, Reason::Signed)) } Err(reason) => { @@ -674,7 +936,7 @@ impl Signer { async fn policy_digest( &self, - request: &PageRequest, + page_url: &str, services: &RuntimeServices, kv: &dyn PlatformKvStore, work: &mut Deadline<'_>, @@ -709,7 +971,7 @@ impl Signer { Ok(None) => {} } - let reason = match self.request_policy(request, services, work).await { + let reason = match self.request_policy(page_url, services, work).await { Ok(policy_digest) => { let entry = PolicyEntry::Digest { policy_digest: policy_digest.clone(), @@ -758,13 +1020,12 @@ impl Signer { async fn request_policy( &self, - request: &PageRequest, + page_url: &str, services: &RuntimeServices, work: &mut Deadline<'_>, ) -> Result { let mut url = url::Url::parse(&self.policy_url).map_err(|_| Reason::RequestBuild)?; - url.query_pairs_mut() - .append_pair("page_url", &request.page_url); + url.query_pairs_mut().append_pair("page_url", page_url); let body = self .fetch( services, @@ -834,7 +1095,15 @@ impl Signer { Ok((plan, parsed.manifest_url)) } - /// One bounded HTTP call; never retried. + /// The worker token header value, marked sensitive so no log shows it. + fn worker_token(&self) -> Result { + let mut token = HeaderValue::from_str(self.config.worker_token.expose().trim()) + .map_err(|_| Reason::RequestBuild)?; + token.set_sensitive(true); + Ok(token) + } + + /// One bounded JSON HTTP call; never retried. async fn fetch( &self, services: &RuntimeServices, @@ -844,36 +1113,17 @@ impl Signer { max_response_bytes: usize, work: &mut Deadline<'_>, ) -> Result { - let mut token = HeaderValue::from_str(self.config.worker_token.expose().trim()) - .map_err(|_| Reason::RequestBuild)?; - token.set_sensitive(true); let http_request = Request::builder() .method(method) .uri(url) .header(header::CONTENT_TYPE, "application/json") - .header(WORKER_TOKEN_HEADER, token) + .header(WORKER_TOKEN_HEADER, self.worker_token()?) .body(body) .map_err(|_| Reason::RequestBuild)?; - let backend = ensure_integration_backend_with_timeout( - services, - &self.config.api_base_url, - INTEGRATION_ID, - Duration::from_millis(self.config.timeout_ms), - ) - .map_err(|_| Reason::Backend)?; - let response_limit = max_response_bytes.saturating_add(1); - let platform_request = PlatformHttpRequest::new(http_request, backend) - .with_cache_bypass() - .with_max_response_body_bytes(response_limit); - let response = match work - .race(services.http_client().send(platform_request)) - .await - { - None => return Err(Reason::Timeout), - Some(Err(_)) => return Err(Reason::HttpError), - Some(Ok(response)) => response.response, - }; + let response = self + .send_api(services, http_request, response_limit, work) + .await?; let status = response.status(); if status != StatusCode::OK { return Err(match status.as_u16() { @@ -896,6 +1146,35 @@ impl Signer { } } + /// Send one request to the API backend, bounded by the first-byte + /// timeout and `work`, reading at most `response_limit` body bytes. + async fn send_api( + &self, + services: &RuntimeServices, + http_request: Request, + response_limit: usize, + work: &mut Deadline<'_>, + ) -> Result, Reason> { + let backend = ensure_integration_backend_with_timeout( + services, + &self.config.api_base_url, + INTEGRATION_ID, + Duration::from_millis(self.config.timeout_ms), + ) + .map_err(|_| Reason::Backend)?; + let platform_request = PlatformHttpRequest::new(http_request, backend) + .with_cache_bypass() + .with_max_response_body_bytes(response_limit); + match work + .race(services.http_client().send(platform_request)) + .await + { + None => Err(Reason::Timeout), + Some(Err(_)) => Err(Reason::HttpError), + Some(Ok(response)) => Ok(response.response), + } + } + /// Read a fresh entry. Expired entries are misses; in TTL-less write mode /// they are also deleted, best effort, so the store cleans itself. async fn read( @@ -1510,6 +1789,7 @@ mod tests { PageRequest { method: Method::GET, page_url: PAGE_URL.to_owned(), + removed_prefix: String::new(), response_status: 200, origin_response_is_private: false, } @@ -1567,7 +1847,7 @@ mod tests { return html.to_owned(); }; futures::executor::block_on(async { - let mut processor = document_processor(factory, request); + let mut processor = document_processor(factory, request, None); let (first, second) = html.as_bytes().split_at(html.len() / 2); let mut output = processor .process_chunk(first, false) @@ -2327,8 +2607,19 @@ mod tests { let http = Arc::new(StubHttpClient::new()); let services = services(http, &kv); - let result = - futures::executor::block_on(signer.resolve(&request(), &services, ARTICLE, &article)); + let result = futures::executor::block_on(async { + let mut work = Deadline::Unbounded; + signer + .resolve( + &request(), + &services, + ARTICLE, + &article, + POLICY_DIGEST, + &mut work, + ) + .await + }); assert_eq!( result.expect_err("alignment should fail"), @@ -2859,6 +3150,14 @@ mod tests { }), Gate::ExcludedPath, ), + ( + ("excluded path under an addressing prefix", &signer, |r| { + r.page_url = + "https://publisher.example.com/www.site.example/wp-admin/post".to_owned(); + r.removed_prefix = "/www.site.example".to_owned(); + }), + Gate::ExcludedPath, + ), ( ("asset suffix", &signer, |r| { r.page_url = "https://publisher.example.com/feed.XML".to_owned(); diff --git a/crates/provenance/encypher/src/processor/media.rs b/crates/provenance/encypher/src/processor/media.rs new file mode 100644 index 0000000000..842acd75ec --- /dev/null +++ b/crates/provenance/encypher/src/processor/media.rs @@ -0,0 +1,893 @@ +//! Media signing through the Encypher media endpoint. +//! +//! One source's exact bytes are sent with the server policy the edge holds, +//! and the answer is kept only after [`media_api::verify_signed_response`] +//! accepts it. Results are cached in the runtime KV store under a namespace +//! that binds the credential context, the policy digest, the delivery +//! address, the source reference, its declared and verified type, its bytes' +//! digest and whether a `Link` can be delivered, so a changed policy, +//! credential or source never reuses another's signature. The replay key the +//! endpoint requires is derived from the same namespace, so a repeated call +//! is answered with the identical bytes. +//! +//! Two callers use it: core's binary delivery seam, through [`EncypherMedia`], +//! and the page and stylesheet processors for embedded `data:` URIs and +//! inline SVG, through [`Signer::sign_embedded`]. Core keeps the signed +//! representations it delivers under [`MediaSigner::namespace`], which the +//! module answers only after a policy check no older than +//! `policy_ttl_seconds`, so a revoked credential or changed policy stops +//! every kept copy from being served. + +use std::sync::Arc; +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use http::{HeaderMap, HeaderName, HeaderValue, Method, Request, StatusCode, header}; +use serde::{Deserialize, Serialize}; +use trusted_server_core::media_signing::{ + LargeMediaJob, LargeMediaSigning, MediaJob, MediaNamespacing, MediaSigner, MediaSigning, + MediaSigningFailure, SignedMedia as DeliveredMedia, +}; +use trusted_server_core::middleware::ResponseHeader; +use trusted_server_core::platform::RuntimeServices; + +use super::{ + Deadline, Expiring, PageRequest, Reason, SKIP_WRITE_RESERVE, Signer, Surface, + WORKER_TOKEN_HEADER, is_excluded_prefix, unix_now, +}; +use crate::MODULE; +use crate::data_uri::DataUri; +use crate::embedded::{self, Scan}; +use crate::media_api::{ + self, Disposition, Expected, MediaMetadata, ResponseError, SignedMedia, SourceMetadata, + canonical_media_type, safe_source_reference, sha256_hex, valid_origin_host, +}; +use crate::svg_equivalence; + +/// Verified signed media. Entries carry their own `expires_at`. +const MEDIA_KEY_PREFIX: &str = "encypher:media:v1"; +/// Media failures, separate so a delayed failure cannot replace a result. +const MEDIA_SKIP_KEY_PREFIX: &str = "encypher:media-skip:v1"; +/// A fresh replay-key salt, written when the endpoint no longer holds or +/// refuses the result for the previous key. +const MEDIA_REPLAY_KEY_PREFIX: &str = "encypher:media-replay:v1"; +const SVG_MEDIA_TYPE: &str = "image/svg+xml"; + +/// The media types the signer signs: images, audio, video and fonts, +/// whatever the path they are delivered on. Whether a format is signable is +/// decided by its verified bytes at the endpoint, which answers an +/// unsupported one `422`, delivered unsigned. +const SIGNABLE_TYPE_PREFIXES: &[&str] = &["image/", "audio/", "video/", "font/"]; + +/// The media type of one segment of a fragmented ISO BMFF stream, as ISO/IEC +/// 14496-12 registers it. A segment's provenance is bound through its +/// stream's init segment, as with C2PA verifiable segment info, so it is +/// never signed alone and reaches the reader as the origin sent it. +const MEDIA_SEGMENT_TYPE: &str = "video/iso.segment"; + +const INSTANCE_ID_HEADER: HeaderName = HeaderName::from_static("x-encypher-manifest-instance-id"); +const DOCUMENT_ID_HEADER: HeaderName = HeaderName::from_static("x-encypher-document-id"); +const VERIFICATION_URL_HEADER: HeaderName = HeaderName::from_static("x-encypher-verification-url"); + +/// One media source to sign, as it is delivered. +struct MediaSource<'a> { + /// The reader-facing address the media is delivered on or in. + page_url: &'a str, + /// A safe canonical address of the source, or none. + reference: Option<&'a str>, + /// The hostname the source was fetched from, or none for media embedded + /// in a document. + origin_host: Option<&'a str>, + /// The declared type, sent as a hint. + mime: &'a str, + /// The bare type the signed bytes must keep. + media_type: &'a str, + bytes: &'a [u8], + sha256: &'a str, + /// Whether every response of the representation can carry a `Link`. + link_deliverable: bool, + /// The bytes are the platform image optimizer's answer, so the endpoint + /// must refuse them unless they carry an intact embedded manifest, the + /// optimizer's own record of its input and what it did. + require_embedded_manifest: bool, +} + +/// A cached media result. +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum MediaEntry { + /// The verified answer: its base64 bytes and its signing headers, which + /// are verified again on every read. + Signed { + bytes: String, + headers: Vec<(String, String)>, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Skip { + reason: Reason, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Replay { + salt: String, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, +} + +impl Expiring for MediaEntry { + fn expires_at(&self) -> u64 { + match self { + Self::Signed { expires_at, .. } + | Self::Skip { expires_at, .. } + | Self::Replay { expires_at, .. } => *expires_at, + } + } + + fn ttl_less(&self) -> bool { + match self { + Self::Signed { ttl_less, .. } + | Self::Skip { ttl_less, .. } + | Self::Replay { ttl_less, .. } => *ttl_less, + } + } + + fn mark_ttl_less(&mut self) { + match self { + Self::Signed { ttl_less, .. } + | Self::Skip { ttl_less, .. } + | Self::Replay { ttl_less, .. } => *ttl_less = true, + } + } +} + +impl MediaEntry { + fn signed(signed: &SignedMedia, headers: &HeaderMap, expires_at: u64) -> Self { + let headers = media_api::SIGNING_HEADERS + .iter() + .filter_map(|name| { + let value = headers.get(*name)?.to_str().ok()?; + Some(((*name).to_owned(), value.to_owned())) + }) + .collect(); + Self::Signed { + bytes: STANDARD.encode(&signed.bytes), + headers, + expires_at, + ttl_less: false, + } + } +} + +/// A cached answer, verified as if it had just arrived. +fn verify_cached( + bytes: &str, + headers: &[(String, String)], + expected: &Expected<'_>, +) -> Result { + let body = STANDARD + .decode(bytes) + .map_err(|_| Reason::CachedEntryInvalid)?; + let mut map = HeaderMap::with_capacity(headers.len()); + for (name, value) in headers { + let name = + HeaderName::from_bytes(name.as_bytes()).map_err(|_| Reason::CachedEntryInvalid)?; + let value = HeaderValue::from_str(value).map_err(|_| Reason::CachedEntryInvalid)?; + map.append(name, value); + } + media_api::verify_signed_response(&map, Bytes::from(body), expected) + .map_err(|_| Reason::CachedEntryInvalid) +} + +fn response_reason(error: ResponseError) -> Reason { + match error { + ResponseError::ContentType => Reason::SignedTypeMismatch, + ResponseError::PolicyDigest => Reason::PolicyDigestMismatch, + ResponseError::SignedDigest => Reason::SignedDigestMismatch, + ResponseError::Malformed => Reason::InvalidSignedResponse, + ResponseError::LinkUndeliverable => Reason::ManifestUndeliverable, + ResponseError::Unsigned => Reason::UnsignedResponse, + } +} + +/// The reason for an error answer, by its status and failure code. +fn status_reason(status: StatusCode, code: Option<&str>) -> Reason { + match status.as_u16() { + 403 | 404 => Reason::Forbidden, + 409 if code == Some("SIGNING_IDEMPOTENCY_CONFLICT") => Reason::IdempotencyConflict, + 409 => Reason::PolicyChanged, + 410 => Reason::ResultPurged, + 413 => Reason::PayloadTooLarge, + 422 => Reason::Unprocessable, + 429 => Reason::RateLimited, + 500..=599 => Reason::ServerError, + _ => Reason::UnexpectedStatus, + } +} + +fn log_media(signed: bool, reason: Reason) { + log::info!( + "encypher: media outcome={} reason={}", + if signed { "signed" } else { "unsigned" }, + reason.code() + ); +} + +/// Reports every media item of `scan` unsigned for `reason`. +pub(super) fn log_unsigned(scan: &Scan, reason: Reason) { + for refused in &scan.refused { + log_media(false, *refused); + } + for _ in 0..scan.found.len() + scan.svgs.len() { + log_media(false, reason); + } +} + +/// The document an embedded media pass works for. +pub(super) struct EmbeddedJob<'a> { + pub(super) request: &'a PageRequest, + pub(super) surface: Surface, + pub(super) policy_digest: &'a str, +} + +/// One item's outcome in an embedded pass: signed, from the cache or fresh, +/// or unsigned with why. +type Outcome = Result; + +impl Signer { + fn max_media_bytes(&self) -> usize { + self.config.max_media_bytes + } + + /// The cache namespace of `source` under `policy_digest`: every input the + /// signed result depends on. + fn media_namespace(&self, source: &MediaSource<'_>, policy_digest: &str) -> String { + sha256_hex( + format!( + "{}\n{policy_digest}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}", + self.cache_context, + source.page_url, + source.reference.unwrap_or_default(), + source.origin_host.unwrap_or_default(), + source.mime, + source.media_type, + source.sha256, + source.link_deliverable, + source.require_embedded_manifest + ) + .as_bytes(), + ) + } + + /// Sign one source, from the cache when a verified result is held. + /// Returns the result and whether it was signed now or read back. A call + /// to the endpoint is made only while `calls` is below + /// `self.config.max_media_sign_calls_per_document`, and counted there. + async fn sign_media( + &self, + services: &RuntimeServices, + source: &MediaSource<'_>, + policy_digest: &str, + work: &mut Deadline<'_>, + calls: &mut usize, + ) -> Result<(SignedMedia, Reason), Reason> { + if source.bytes.len() > self.max_media_bytes() { + return Err(Reason::MediaTooLarge); + } + // Once the budget is spent no call is made and no skip written, so + // the reserve stays for the document's own writes. + if work.expired() { + return Err(Reason::Timeout); + } + let kv = services.kv_store(); + let namespace = self.media_namespace(source, policy_digest); + let key = format!("{MEDIA_KEY_PREFIX}:{namespace}"); + let skip_key = format!("{MEDIA_SKIP_KEY_PREFIX}:{namespace}"); + let expected = Expected { + policy_digest, + media_type: source.media_type, + unsigned_sha256: source.sha256, + link_deliverable: source.link_deliverable, + }; + let outcome = match self.read::(kv, &key, work).await { + Ok(Some(MediaEntry::Signed { bytes, headers, .. })) => { + return match verify_cached(&bytes, &headers, &expected) { + Ok(signed) => Ok((signed, Reason::CacheHit)), + Err(reason) => { + self.delete_invalid_entry(kv, &key, work).await; + Err(reason) + } + }; + } + Ok(Some(_)) => { + self.delete_invalid_entry(kv, &key, work).await; + return Err(Reason::CachedEntryInvalid); + } + Err(Reason::Timeout) => Err(Reason::Timeout), + Err(reason) => return Err(reason), + Ok(None) => match self.read::(kv, &skip_key, work).await { + Ok(Some(MediaEntry::Skip { .. })) => return Err(Reason::CachedSkip), + Ok(Some(_)) => { + self.delete_invalid_entry(kv, &skip_key, work).await; + return Err(Reason::CachedEntryInvalid); + } + Err(Reason::Timeout) => Err(Reason::Timeout), + Err(reason) => return Err(reason), + Ok(None) if *calls >= self.config.max_media_sign_calls_per_document => { + return Err(Reason::Deferred); + } + Ok(None) => { + *calls += 1; + self.request_media(services, source, policy_digest, &namespace, &expected, work) + .await + } + }, + }; + match outcome { + Ok((signed, headers)) => { + // A result write that runs out of time is abandoned; the media + // is still signed. + let entry = MediaEntry::signed( + &signed, + &headers, + Self::expires_at(self.config.plan_ttl_seconds), + ); + self.write(kv, &key, entry, self.config.plan_ttl_seconds, work) + .await; + Ok((signed, Reason::Signed)) + } + Err(reason @ (Reason::ResultPurged | Reason::IdempotencyConflict)) => { + // The next view asks again under a new replay key; no skip, + // and never a retry within this view. + let entry = MediaEntry::Replay { + salt: format!("{}", unix_now()), + expires_at: Self::expires_at(self.config.plan_ttl_seconds), + ttl_less: false, + }; + let replay_key = format!("{MEDIA_REPLAY_KEY_PREFIX}:{namespace}"); + self.write( + kv, + &replay_key, + entry, + self.config.plan_ttl_seconds, + &mut Self::skip_deadline(services), + ) + .await; + Err(reason) + } + Err(reason) => { + let mut deadline = Self::skip_deadline(services); + if matches!(reason, Reason::PolicyChanged | Reason::PolicyDigestMismatch) { + self.evict_policy(kv, &mut deadline).await; + } + let ttl = self.config.negative_ttl_seconds; + let entry = MediaEntry::Skip { + reason, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write(kv, &skip_key, entry, ttl, &mut deadline).await; + Err(reason) + } + } + } + + /// Call the media endpoint once with the source's exact bytes. Every + /// failure maps to a reason code. + async fn request_media( + &self, + services: &RuntimeServices, + source: &MediaSource<'_>, + policy_digest: &str, + namespace: &str, + expected: &Expected<'_>, + work: &mut Deadline<'_>, + ) -> Result<(SignedMedia, HeaderMap), Reason> { + let kv = services.kv_store(); + let replay_key = format!("{MEDIA_REPLAY_KEY_PREFIX}:{namespace}"); + let salt = match self.read::(kv, &replay_key, work).await { + Ok(Some(MediaEntry::Replay { salt, .. })) => salt, + Ok(Some(_)) => { + self.delete_invalid_entry(kv, &replay_key, work).await; + return Err(Reason::CachedEntryInvalid); + } + Ok(None) => String::new(), + Err(reason) => return Err(reason), + }; + let idempotency_key = sha256_hex(format!("{namespace}\n{salt}").as_bytes()); + let metadata = serde_json::to_vec(&MediaMetadata::unmodified( + source.page_url, + policy_digest, + SourceMetadata { + reference: source.reference, + origin_host: source.origin_host, + mime: source.mime, + sha256: source.sha256, + byte_length: source.bytes.len(), + require_embedded_manifest: source.require_embedded_manifest, + }, + source.link_deliverable, + )) + .map_err(|_| Reason::RequestBuild)?; + let multipart = media_api::multipart(&metadata, source.bytes); + if multipart.body.len() > media_api::MAX_REQUEST_BYTES { + return Err(Reason::PayloadTooLarge); + } + let http_request = Request::builder() + .method(Method::POST) + .uri(&self.media_sign_url) + .header(header::CONTENT_TYPE, multipart.content_type) + .header(WORKER_TOKEN_HEADER, self.worker_token()?) + .header(media_api::IDEMPOTENCY_KEY_HEADER, idempotency_key) + .body(EdgeBody::from_bytes(multipart.body)) + .map_err(|_| Reason::RequestBuild)?; + let max_body = source + .bytes + .len() + .saturating_add(media_api::MAX_MANIFEST_OVERHEAD_BYTES); + let response_limit = max_body.saturating_add(1); + let response = self + .send_api(services, http_request, response_limit, work) + .await?; + let status = response.status(); + let headers = response.headers().clone(); + if status != StatusCode::OK { + let code = work + .race( + response + .into_body() + .into_bytes_bounded(media_api::MAX_ERROR_BODY_BYTES), + ) + .await + .and_then(Result::ok) + .and_then(|body| media_api::error_code(&body)); + log::info!( + "encypher: media sign answered status={} code={}", + status.as_u16(), + code.as_deref().unwrap_or("none") + ); + return Err(status_reason(status, code.as_deref())); + } + let body = match work + .race(response.into_body().into_bytes_bounded(response_limit)) + .await + { + None => return Err(Reason::Timeout), + Some(Err(_)) => return Err(Reason::ResponseTooLarge), + Some(Ok(body)) => body, + }; + if body.len() > max_body { + return Err(Reason::ResponseTooLarge); + } + let signed = + media_api::verify_signed_response(&headers, body, expected).map_err(response_reason)?; + Ok((signed, headers)) + } + + /// Sign the editable media embedded in `document`: each `data:` URI + /// found, then each inline SVG last as the equivalent standalone document. + /// The scan excludes entire original provenance-bearing or uncertain SVG + /// ranges before either pass, so child signing cannot invalidate them. + /// Returns the new document, or `None` when nothing in it was signed. + pub(super) async fn sign_embedded( + &self, + services: &RuntimeServices, + job: &EmbeddedJob<'_>, + document: &str, + scan: &Scan, + work: &mut Deadline<'_>, + ) -> Option { + for reason in &scan.refused { + log_media(false, *reason); + } + let mut outcomes: Vec = Vec::new(); + let mut edits: Vec<(usize, usize, String)> = Vec::new(); + let mut signed_sources: Vec<(String, String, Result)> = Vec::new(); + let mut calls = 0_usize; + for found in &scan.found { + let Some(uri) = document + .get(found.start..found.end) + .and_then(DataUri::parse) + else { + continue; + }; + let Ok(bytes) = uri.decode() else { + outcomes.push(Err(Reason::MalformedDataUri)); + continue; + }; + let media_type = uri.media_type(); + let sha256 = sha256_hex(&bytes); + let cached = signed_sources + .iter() + .find(|(sha, kind, _)| *sha == sha256 && *kind == media_type) + .map(|(_, _, result)| result.clone()); + let result = match cached { + Some(result) => result.map(|signed| (signed, Reason::CacheHit)), + None => { + let source = MediaSource { + page_url: &job.request.page_url, + reference: None, + origin_host: None, + mime: uri.declared_type(), + media_type: &media_type, + bytes: &bytes, + sha256: &sha256, + link_deliverable: false, + require_embedded_manifest: false, + }; + let result = self + .sign_media(services, &source, job.policy_digest, work, &mut calls) + .await + .and_then(|(signed, reason)| { + if media_type == SVG_MEDIA_TYPE { + embedded::verify_svg_carrier(&bytes, &signed.bytes)?; + } + Ok((signed, reason)) + }); + signed_sources.push(( + sha256.clone(), + media_type.clone(), + result + .as_ref() + .map(|(signed, _)| signed.clone()) + .map_err(|reason| *reason), + )); + result + } + }; + match result { + Ok((signed, reason)) => { + edits.push((found.start, found.end, uri.encode(&signed.bytes))); + outcomes.push(Ok(reason)); + } + Err(reason) => outcomes.push(Err(reason)), + } + } + let mut output = None; + if !edits.is_empty() { + output = embedded::splice(document, &edits, self.max_document_bytes); + if output.is_none() { + revoke(&mut outcomes); + } + } + if job.surface == Surface::Page { + let current = output.as_deref().unwrap_or(document); + let svgs = if output.is_some() { + embedded::scan_html(current, None).svgs + } else { + scan.svgs.clone() + }; + let mut svg_outcomes = Vec::with_capacity(svgs.len()); + let mut svg_edits = Vec::new(); + for (start, end) in svgs { + let Some(fragment) = current.get(start..end) else { + continue; + }; + match self + .sign_inline_svg(services, job, fragment, work, &mut calls) + .await + { + Ok((signed, reason)) => { + svg_edits.push((start, end, signed)); + svg_outcomes.push(Ok(reason)); + } + Err(reason) => svg_outcomes.push(Err(reason)), + } + } + if !svg_edits.is_empty() { + match embedded::splice(current, &svg_edits, self.max_document_bytes) { + Some(signed) => output = Some(signed), + None => revoke(&mut svg_outcomes), + } + } + outcomes.extend(svg_outcomes); + } + for outcome in &outcomes { + match outcome { + Ok(reason) => log_media(true, *reason), + Err(reason) => log_media(false, *reason), + } + } + output + } + + /// Sign one inline SVG as the standalone document + /// [`signable_svg`](svg_equivalence::signable_svg) proves equivalent to + /// the browser's tree: the original bytes where they already are one, + /// else the browser's tree serialized as XML. Those bytes are hashed, + /// sent, cached and compared with the answer, and the result is kept + /// only when the signer added its manifest and changed nothing else. + async fn sign_inline_svg( + &self, + services: &RuntimeServices, + job: &EmbeddedJob<'_>, + fragment: &str, + work: &mut Deadline<'_>, + calls: &mut usize, + ) -> Result<(String, Reason), Reason> { + let svg = svg_equivalence::signable_svg(fragment, self.max_media_bytes())?; + let bytes = svg.as_bytes(); + let sha256 = sha256_hex(bytes); + let source = MediaSource { + page_url: &job.request.page_url, + reference: None, + origin_host: None, + mime: SVG_MEDIA_TYPE, + media_type: SVG_MEDIA_TYPE, + bytes, + sha256: &sha256, + link_deliverable: false, + require_embedded_manifest: false, + }; + let (signed, reason) = self + .sign_media(services, &source, job.policy_digest, work, calls) + .await?; + embedded::verify_svg_carrier(bytes, &signed.bytes)?; + let signed = + String::from_utf8(signed.bytes.to_vec()).map_err(|_| Reason::InlineSvgAltered)?; + Ok((signed, reason)) + } + + /// The namespace core keeps this credential's signed representations + /// under: the module version, the policy digest the service answers now, + /// and the credential context hash. Never a secret. The policy is asked + /// for the claim's root, being the policy of the credential whatever the + /// asset, and asked live on every call, never from the article path's + /// cache, so a revoked credential or changed policy stops every kept copy + /// at once. + async fn delivery_namespace(&self, services: &RuntimeServices) -> Result { + let mut work = Deadline::new( + services.timer(), + Duration::from_millis(self.config.max_hold_ms).saturating_sub(SKIP_WRITE_RESERVE), + ); + work.start().await; + let claim_root = format!("https://{}/", self.config.domain); + let policy_digest = self + .request_policy(&claim_root, services, &mut work) + .await?; + Ok(self.kept_namespace(&policy_digest)) + } + + /// The namespace of signed representations made under `policy_digest` + /// with this credential, as [`Self::delivery_namespace`] answers it. + pub(super) fn kept_namespace(&self, policy_digest: &str) -> String { + format!("{NAMESPACE_VERSION}:{policy_digest}:{}", self.cache_context) + } + + /// Sign one binary representation core captured, for every response of + /// it: the reader-facing address must be on the claimed domain, the + /// bytes the ones core hashed. + async fn sign_delivery( + &self, + services: &RuntimeServices, + job: &MediaJob<'_>, + ) -> Result { + let reader_url = url::Url::parse(job.reader_url).map_err(|_| Reason::OutsideClaim)?; + let host = reader_url.host_str().ok_or(Reason::OutsideClaim)?; + if !matches!(reader_url.scheme(), "https" | "http") + || !self.claims_host(host) + || is_excluded_prefix(reader_url.path()) + { + return Err(Reason::OutsideClaim); + } + if job.source.len() > self.max_media_bytes() { + return Err(Reason::MediaTooLarge); + } + let sha256 = sha256_hex(job.source); + if sha256 != job.source_sha256 { + return Err(Reason::SourceDigestMismatch); + } + // The host core fetched from, which the endpoint checks the claim's + // managed origins and the reference against. + let origin_host = job + .source_origin_host + .filter(|host| valid_origin_host(host)) + .ok_or(Reason::InvalidOriginHost)?; + let media_type = canonical_media_type(job.content_type); + // No address serves the optimizer's answer, so it is bound by its + // own bytes and never by the address the optimizer read. + let reference = job.source_reference.filter(|reference| { + !job.optimizer_output && safe_source_reference(reference, &self.config.domain) + }); + let mut work = Deadline::new( + services.timer(), + Duration::from_millis(self.config.max_hold_ms).saturating_sub(SKIP_WRITE_RESERVE), + ); + work.start().await; + // Asked live, as for the namespace: a cached digest could outlive a + // revocation and reuse a signed result under it. + let policy_digest = self + .request_policy(job.reader_url, services, &mut work) + .await?; + let source = MediaSource { + page_url: job.reader_url, + reference, + origin_host: Some(origin_host), + mime: job.content_type, + media_type: &media_type, + bytes: job.source, + sha256: &sha256, + link_deliverable: true, + require_embedded_manifest: job.optimizer_output, + }; + // One representation, one call at most. + let mut calls = 0_usize; + let (signed, reason) = self + .sign_media(services, &source, &policy_digest, &mut work, &mut calls) + .await?; + if media_type == SVG_MEDIA_TYPE && signed.disposition == Disposition::Embedded { + embedded::verify_svg_carrier(job.source, &signed.bytes)?; + } + let mut headers: Vec = Vec::with_capacity(4); + if signed.disposition == Disposition::External + && let Some(link) = signed.link() + { + headers.push(( + header::LINK, + HeaderValue::from_str(&link).map_err(|_| Reason::InvalidSignedResponse)?, + )); + } + for (name, value) in [ + (INSTANCE_ID_HEADER, &signed.instance_id), + (DOCUMENT_ID_HEADER, &signed.document_id), + (VERIFICATION_URL_HEADER, &signed.verification_url), + ] { + headers.push(( + name, + HeaderValue::from_str(value).map_err(|_| Reason::InvalidSignedResponse)?, + )); + } + log_media(true, reason); + Ok(DeliveredMedia { + bytes: signed.bytes, + content_type: signed.content_type, + headers, + // The policy this representation was signed under, so core keeps + // it where only that policy finds it. + namespace: self.kept_namespace(&policy_digest), + }) + } +} + +/// Marks every signed outcome of a batch whose output would exceed the +/// document budget as unsigned: none of its edits is applied. +fn revoke(outcomes: &mut [Outcome]) { + for outcome in outcomes.iter_mut() { + if outcome.is_ok() { + *outcome = Err(Reason::OutputTooLarge); + } + } +} + +/// The version of the module's signed representations, part of every +/// namespace core keeps them under. +const NAMESPACE_VERSION: &str = concat!("media-v1:", env!("CARGO_PKG_VERSION")); + +/// The module's binary media signer, which core runs on publisher and asset +/// proxy responses of the media it captures. +pub(crate) struct EncypherMedia { + signer: Arc, +} + +impl EncypherMedia { + pub(crate) fn new(signer: Arc) -> Self { + Self { signer } + } +} + +impl std::fmt::Debug for EncypherMedia { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("EncypherMedia") + .finish_non_exhaustive() + } +} + +impl MediaSigner for EncypherMedia { + fn signer_id(&self) -> &'static str { + MODULE + } + + /// Every path but an API, feed, admin or well-known path: media is + /// recognized by the origin's media type, not by a suffix. + fn handles_path(&self, path: &str) -> bool { + !is_excluded_prefix(path) + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type != MEDIA_SEGMENT_TYPE + && SIGNABLE_TYPE_PREFIXES + .iter() + .any(|prefix| media_type.starts_with(prefix) && media_type.len() > prefix.len()) + } + + fn max_source_bytes(&self) -> usize { + self.signer.max_media_bytes() + } + + /// Larger media, up to this, is signed through the prepared lane: hashed + /// where it is delivered and never uploaded. + fn max_large_source_bytes(&self) -> u64 { + self.signer.max_large_media_bytes() + } + + /// The one budget for a large source: core's whole-source hash and the + /// prepared lane together. + fn max_large_media_duration(&self) -> Duration { + self.signer.max_large_media_duration() + } + + fn sign_large<'a>( + &'a self, + services: &'a RuntimeServices, + job: &'a LargeMediaJob<'a>, + ) -> LargeMediaSigning<'a> { + Box::pin(async move { + match self.signer.sign_large(services, job).await { + Ok(signed) => { + log_media(true, Reason::Signed); + Ok(signed) + } + Err(refusal) => { + // A source with its own embedded credentials is delivered + // as it is: its provenance is kept, it is not unsigned + // by failure and not newly signed. + log::info!( + "encypher: media outcome={} lane=large reason={}", + if refusal.retains_provenance() { + "retained" + } else { + "unsigned" + }, + refusal.code() + ); + Err(MediaSigningFailure { + reason: refusal.code(), + }) + } + } + }) + } + + fn namespace<'a>(&'a self, services: &'a RuntimeServices) -> MediaNamespacing<'a> { + Box::pin(async move { + self.signer + .delivery_namespace(services) + .await + .map_err(|reason| { + log_media(false, reason); + MediaSigningFailure { + reason: reason.code(), + } + }) + }) + } + + fn sign<'a>( + &'a self, + services: &'a RuntimeServices, + job: &'a MediaJob<'a>, + ) -> MediaSigning<'a> { + Box::pin(async move { + self.signer + .sign_delivery(services, job) + .await + .map_err(|reason| { + log_media(false, reason); + MediaSigningFailure { + reason: reason.code(), + } + }) + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/provenance/encypher/src/processor/media/tests.rs b/crates/provenance/encypher/src/processor/media/tests.rs new file mode 100644 index 0000000000..fb8de7cbfa --- /dev/null +++ b/crates/provenance/encypher/src/processor/media/tests.rs @@ -0,0 +1,1188 @@ +use std::collections::HashMap; +use std::sync::Mutex; + +use edgezero_core::key_value_store::KvPage; +use futures::executor::block_on; +use serde_json::json; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use trusted_server_core::platform::{KvError, PlatformHttpClient, PlatformKvStore}; + +use super::*; +use crate::EncypherConfig; + +const TOKEN: &str = "cwk_test_worker_token_value"; +const POLICY_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const PAGE_URL: &str = "https://publisher.example.com/news/story"; +const SIGN_URL: &str = "https://api.example.com/api/v1/public/cdn/media/sign"; +const SOURCE: &[u8] = b"\x89PNG\r\n\x1a\n-source-pixels"; +const SIGNED: &[u8] = b"\x89PNG\r\n\x1a\n-source-pixels-caBX-jumb-c2pa-manifest"; +const MANIFEST: &str = "QUJD"; + +/// A runtime KV store held in memory, recording deletes. +#[derive(Default)] +struct MemoryKv { + entries: Mutex>, + deletes: Mutex>, +} + +impl MemoryKv { + fn keys_with(&self, prefix: &str) -> Vec { + self.entries + .lock() + .expect("should lock entries") + .keys() + .filter(|key| key.starts_with(prefix)) + .cloned() + .collect() + } + + fn get(&self, key: &str) -> Option { + self.entries + .lock() + .expect("should lock entries") + .get(key) + .cloned() + } + + fn set(&self, key: &str, value: Bytes) { + self.entries + .lock() + .expect("should lock entries") + .insert(key.to_owned(), value); + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + Ok(self.get(key)) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.set(key, value); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + _ttl: Duration, + ) -> Result<(), KvError> { + self.set(key, value); + Ok(()) + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.entries + .lock() + .expect("should lock entries") + .remove(key); + self.deletes + .lock() + .expect("should lock deletes") + .push(key.to_owned()); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(KvPage::default()) + } +} + +fn signer() -> Arc { + let config: EncypherConfig = serde_json::from_value(json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": TOKEN, + })) + .expect("should parse config"); + Arc::new(Signer::new(config, 5_000_000)) +} + +fn services(http: &Arc, kv: &Arc) -> RuntimeServices { + build_services_with_http_client(Arc::clone(http) as Arc) + .with_kv_store(Arc::clone(kv) as Arc) +} + +/// The signing headers of an embedded answer for `body`. +fn answer(body: &[u8], content_type: &str) -> Vec<(String, String)> { + [ + ("content-type", content_type.to_owned()), + ("x-encypher-policy-digest", POLICY_DIGEST.to_owned()), + ("x-encypher-signed-sha256", sha256_hex(body)), + ("x-encypher-manifest-instance-id", "xmp:iid:0001".to_owned()), + ("x-encypher-manifest-disposition", "embedded".to_owned()), + ("x-encypher-manifest-link-required", "false".to_owned()), + ( + "x-encypher-verification-url", + "https://verify.example.com/doc-1".to_owned(), + ), + ("x-encypher-document-id", "doc-1".to_owned()), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect() +} + +/// The signing headers of a remote-manifest answer that needs a `Link`. +fn external_answer(body: &[u8]) -> Vec<(String, String)> { + let mut headers: Vec<(String, String)> = answer(body, "image/png") + .into_iter() + .filter(|(name, _)| { + name != "x-encypher-manifest-disposition" && name != "x-encypher-manifest-link-required" + }) + .collect(); + for (name, value) in [ + ("x-encypher-manifest-disposition", "external"), + ("x-encypher-manifest-link-required", "true"), + ( + "x-encypher-manifest-url", + "https://api.example.com/manifests/doc-1", + ), + ("x-encypher-manifest-sha256", POLICY_DIGEST), + ] { + headers.push((name.to_owned(), value.to_owned())); + } + headers +} + +fn png_source(sha256: &str) -> MediaSource<'_> { + MediaSource { + page_url: PAGE_URL, + reference: None, + origin_host: None, + mime: "image/png", + media_type: "image/png", + bytes: SOURCE, + sha256, + link_deliverable: false, + require_embedded_manifest: false, + } +} + +fn sign( + signer: &Signer, + services: &RuntimeServices, + source: &MediaSource<'_>, +) -> Result<(SignedMedia, Reason), Reason> { + block_on(async { + let mut work = Deadline::Unbounded; + let mut calls = 0; + signer + .sign_media(services, source, POLICY_DIGEST, &mut work, &mut calls) + .await + }) +} + +#[test] +fn a_document_obeys_its_configured_call_limit_for_distinct_assets() { + for limit in [3, 12] { + let http = Arc::new(StubHttpClient::new()); + let images: Vec> = (0..=limit) + .map(|index| { + let mut bytes = SOURCE.to_vec(); + bytes.push(u8::try_from(index).expect("should fit")); + bytes + }) + .collect(); + for image in images.iter().take(limit) { + let mut signed = image.clone(); + signed.extend_from_slice(b"-c2pa"); + let headers = answer(&signed, "image/png"); + http.push_response_with_headers(200, signed, headers); + } + let kv = Arc::new(MemoryKv::default()); + let document: String = images + .iter() + .map(|image| { + format!( + "", + STANDARD.encode(image) + ) + }) + .collect(); + let mut config = signer().config.clone(); + config.max_media_sign_calls_per_document = limit; + let signer = Signer::new(config, 5_000_000); + let services = services(&http, &kv); + let request = page_request(); + let scan = embedded::scan_html(&document, None); + let output = block_on(async { + let mut work = Deadline::Unbounded; + signer + .sign_embedded( + &services, + &embedded_job(&request, Surface::Page), + &document, + &scan, + &mut work, + ) + .await + }) + .expect("should sign the items within the configured limit"); + + assert_eq!( + http.recorded_request_methods().len(), + limit, + "should stop calling at the configured bound" + ); + for image in images.iter().take(limit) { + let mut signed = image.clone(); + signed.extend_from_slice(b"-c2pa"); + assert!( + output.contains(&STANDARD.encode(&signed)), + "should deliver every signed distinct asset within the limit" + ); + } + let last = images.last().expect("should have a last image"); + assert!( + output.contains(&STANDARD.encode(last)), + "should leave the deferred item as written" + ); + assert_eq!( + kv.keys_with(MEDIA_SKIP_KEY_PREFIX).len(), + 0, + "a deferred item should not be negatively cached" + ); + } +} + +fn header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(candidate, _)| candidate.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) +} + +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +#[test] +fn signs_exact_bytes_once_then_serves_the_verified_cached_answer() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let signer = signer(); + let sha = sha256_hex(SOURCE); + + let (first, reason) = sign(&signer, &services, &png_source(&sha)).expect("should sign"); + + assert_eq!(reason, Reason::Signed); + assert_eq!(first.bytes.as_ref(), SIGNED, "should keep the signed bytes"); + assert_eq!(http.recorded_request_methods(), ["POST"]); + assert_eq!(http.recorded_request_uris(), [SIGN_URL]); + let headers = &http.recorded_request_headers()[0]; + let key = header(headers, "idempotency-key").expect("should send a replay key"); + assert!( + key.len() == 64 && key.bytes().all(|byte| byte.is_ascii_hexdigit()), + "should derive a hex replay key: {key}" + ); + assert!( + header(headers, "x-encypher-worker-token").is_some(), + "should authenticate with the worker token" + ); + assert!( + header(headers, "content-type") + .is_some_and(|value| value.starts_with("multipart/form-data; boundary=")), + "should send multipart" + ); + let body = &http.recorded_request_bodies()[0]; + assert!(contains(body, SOURCE), "should send the exact source bytes"); + assert!( + contains(body, b"\"reference\":null"), + "should send no reference for media without a safe address" + ); + assert!( + contains(body, b"\"origin_host\":null"), + "should send no origin host for media fetched from no host" + ); + assert!( + contains( + body, + format!("\"expected_policy_digest\":\"{POLICY_DIGEST}\"").as_bytes() + ), + "should send the held policy digest" + ); + + let (second, reason) = sign(&signer, &services, &png_source(&sha)).expect("should sign"); + assert_eq!(reason, Reason::CacheHit); + assert_eq!( + second.bytes.as_ref(), + SIGNED, + "should replay identical bytes" + ); + assert_eq!( + http.recorded_request_methods().len(), + 1, + "should make no second call" + ); +} + +#[test] +fn a_stale_policy_evicts_the_digest_and_caches_a_skip() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 409, + serde_json::to_vec(&json!({ + "success": false, + "error": {"code": "TRUSTED_SERVER_POLICY_CHANGED", "message": "stale"}, + })) + .expect("should encode"), + vec![("content-type", "application/json")], + ); + let kv = Arc::new(MemoryKv::default()); + let signer = signer(); + kv.set(&signer.policy_key, Bytes::from_static(b"{}")); + let services = services(&http, &kv); + let sha = sha256_hex(SOURCE); + + assert_eq!( + sign(&signer, &services, &png_source(&sha)).expect_err("should not sign"), + Reason::PolicyChanged + ); + assert!( + kv.get(&signer.policy_key).is_none(), + "should evict the policy" + ); + assert_eq!( + kv.keys_with(MEDIA_SKIP_KEY_PREFIX).len(), + 1, + "should cache a skip" + ); + assert_eq!( + sign(&signer, &services, &png_source(&sha)).expect_err("should not sign"), + Reason::CachedSkip + ); + assert_eq!( + http.recorded_request_methods().len(), + 1, + "should not call again" + ); +} + +#[test] +fn a_purged_result_is_asked_for_again_under_a_new_key_on_the_next_view() { + for (status, code) in [ + (410, "SIGNING_RESULT_PURGED"), + (409, "SIGNING_IDEMPOTENCY_CONFLICT"), + ] { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + status, + serde_json::to_vec(&json!({"success": false, "error": {"code": code}})) + .expect("should encode"), + ); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let signer = signer(); + let sha = sha256_hex(SOURCE); + + let first = sign(&signer, &services, &png_source(&sha)).expect_err("should not sign"); + assert!( + matches!(first, Reason::ResultPurged | Reason::IdempotencyConflict), + "{code}: should report {first:?}" + ); + assert_eq!( + http.recorded_request_methods().len(), + 1, + "{code}: should not retry" + ); + assert!( + kv.keys_with(MEDIA_SKIP_KEY_PREFIX).is_empty(), + "{code}: no skip" + ); + + let (_, reason) = sign(&signer, &services, &png_source(&sha)).expect("should sign"); + assert_eq!(reason, Reason::Signed); + let headers = http.recorded_request_headers(); + assert_ne!( + header(&headers[0], "idempotency-key"), + header(&headers[1], "idempotency-key"), + "{code}: should mint a new replay key" + ); + } +} + +#[test] +fn a_tampered_cached_answer_is_refused_and_evicted() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let signer = signer(); + let sha = sha256_hex(SOURCE); + sign(&signer, &services, &png_source(&sha)).expect("should sign"); + let key = kv.keys_with(MEDIA_KEY_PREFIX)[0].clone(); + let mut entry: serde_json::Value = + serde_json::from_slice(&kv.get(&key).expect("should hold the answer")).expect("JSON"); + entry["bytes"] = json!(STANDARD.encode(b"\x89PNG\r\n\x1a\nother-c2pa")); + kv.set(&key, Bytes::from(serde_json::to_vec(&entry).expect("JSON"))); + + assert_eq!( + sign(&signer, &services, &png_source(&sha)).expect_err("should refuse"), + Reason::CachedEntryInvalid + ); + assert!(kv.get(&key).is_none(), "should evict the tampered entry"); +} + +#[test] +fn unsigned_or_mismatched_answers_are_never_used_and_are_negatively_cached() { + let other_policy = "b".repeat(64); + let cases = [ + ( + SOURCE.to_vec(), + answer(SOURCE, "image/png"), + Reason::UnsignedResponse, + ), + ( + b"\x89PNG\r\n\x1a\nswapped-c2pa".to_vec(), + answer(SIGNED, "image/png"), + Reason::SignedDigestMismatch, + ), + ( + SIGNED.to_vec(), + answer(SIGNED, "image/jpeg"), + Reason::SignedTypeMismatch, + ), + ( + SIGNED.to_vec(), + answer(SIGNED, "image/png") + .into_iter() + .map(|(name, value)| { + if name == "x-encypher-policy-digest" { + (name, other_policy.clone()) + } else { + (name, value) + } + }) + .collect(), + Reason::PolicyDigestMismatch, + ), + ( + SIGNED.to_vec(), + external_answer(SIGNED), + Reason::ManifestUndeliverable, + ), + ( + SIGNED.to_vec(), + answer(SIGNED, "image/png") + .into_iter() + .map(|(name, value)| { + if name == "x-encypher-verification-url" { + ( + name, + "http://localhost:9000/api/v1/verify/cdnmedia_dev".to_owned(), + ) + } else { + (name, value) + } + }) + .collect(), + Reason::InvalidSignedResponse, + ), + ]; + for (body, headers, expected) in cases { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, body, headers); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let sha = sha256_hex(SOURCE); + + assert_eq!( + sign(&signer(), &services, &png_source(&sha)).expect_err("should not sign"), + expected + ); + assert!( + kv.keys_with(MEDIA_KEY_PREFIX).is_empty(), + "{expected:?}: should cache no answer" + ); + assert_eq!( + kv.keys_with(MEDIA_SKIP_KEY_PREFIX).len(), + 1, + "{expected:?}: should cache the failure" + ); + } +} + +fn embedded_job<'a>(request: &'a PageRequest, surface: Surface) -> EmbeddedJob<'a> { + EmbeddedJob { + request, + surface, + policy_digest: POLICY_DIGEST, + } +} + +fn page_request() -> PageRequest { + PageRequest { + method: Method::GET, + page_url: PAGE_URL.to_owned(), + removed_prefix: String::new(), + response_status: 200, + origin_response_is_private: false, + } +} + +fn sign_document(services: &RuntimeServices, document: &str, surface: Surface) -> Option { + sign_document_with_policies(services, document, surface, None) +} + +fn sign_document_with_policies( + services: &RuntimeServices, + document: &str, + surface: Surface, + policies: Option<&[String]>, +) -> Option { + let request = page_request(); + let scan = match surface { + Surface::Page => embedded::scan_html(document, policies), + Surface::Stylesheet => embedded::scan_stylesheet(document), + }; + block_on(async { + let mut work = Deadline::Unbounded; + signer() + .sign_embedded( + services, + &embedded_job(&request, surface), + document, + &scan, + &mut work, + ) + .await + }) +} + +#[test] +fn embedded_data_uris_are_replaced_by_their_signed_bytes_once_per_source() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let source = format!("data:image/png;base64,{}", STANDARD.encode(SOURCE)); + let signed = format!("data:image/png;base64,{}", STANDARD.encode(SIGNED)); + let document = format!( + "

x

a" + ); + + let output = sign_document(&services(&http, &kv), &document, Surface::Page) + .expect("should sign the page's media"); + + assert_eq!( + output, + format!( + "

x

a" + ), + "should replace each URI and change nothing else" + ); + assert_eq!( + http.recorded_request_methods(), + ["POST"], + "should sign one source once" + ); +} + +#[test] +fn a_failed_embedded_signature_leaves_the_uri_as_written() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 422, + b"{\"error\":{\"code\":\"UNSUPPORTED_MEDIA_TYPE\"}}".to_vec(), + ); + let kv = Arc::new(MemoryKv::default()); + let document = format!( + "", + STANDARD.encode(SOURCE) + ); + + assert_eq!( + sign_document(&services(&http, &kv), &document, Surface::Page), + None, + "should leave the page as written" + ); +} + +#[test] +fn stylesheet_media_is_signed_in_place() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let document = format!( + ".hero{{background:url('data:image/png;base64,{}') no-repeat}}", + STANDARD.encode(SOURCE) + ); + + let output = sign_document(&services(&http, &kv), &document, Surface::Stylesheet) + .expect("should sign the stylesheet's media"); + + assert_eq!( + output, + format!( + ".hero{{background:url('data:image/png;base64,{}') no-repeat}}", + STANDARD.encode(SIGNED) + ) + ); +} + +#[test] +fn inline_svg_is_signed_as_its_exact_bytes_and_kept_only_with_the_writer_s_carrier_alone() { + let root = ""; + let body = ""; + let svg = format!("{root}{body}"); + let document = format!("

x

{svg}

y

"); + let signed_svg = format!("{root}{MANIFEST}{body}"); + let restyled = signed_svg.replace("currentColor", "#000"); + for (answer_svg, expected) in [ + ( + signed_svg.clone(), + Some(format!("

x

{signed_svg}

y

")), + ), + (restyled, None), + ] { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 200, + answer_svg.clone().into_bytes(), + answer(answer_svg.as_bytes(), "image/svg+xml"), + ); + let kv = Arc::new(MemoryKv::default()); + + let output = sign_document(&services(&http, &kv), &document, Surface::Page); + + assert_eq!( + output, expected, + "should keep only a pure manifest insertion" + ); + assert!( + contains(&http.recorded_request_bodies()[0], svg.as_bytes()), + "should send the element's exact bytes" + ); + } +} + +#[test] +fn inline_svg_the_browser_reads_differently_is_signed_as_its_parsed_tree() { + // The browser adds no namespace and keeps the first `viewBox`; what is + // hashed, sent, compared and placed is that tree as standalone XML. + let original = ""; + let serialized = crate::svg_equivalence::signable_svg(original, 1024 * 1024) + .expect("should have an equivalent standalone document") + .into_owned(); + assert_ne!(serialized, original, "should not sign the bytes as written"); + let document = roxmltree::Document::parse(&serialized).expect("should be standalone XML"); + assert_eq!( + document.root_element().attribute("viewBox"), + Some("0 0 2 2"), + "should keep the browser's first viewBox" + ); + // The writer's carrier goes right after the root start tag, whose + // attribute values escape any `>`. + let (root, body) = + serialized.split_at(serialized.find('>').expect("should close the root") + 1); + let signed_svg = format!("{root}{MANIFEST}{body}"); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 200, + signed_svg.clone().into_bytes(), + answer(signed_svg.as_bytes(), "image/svg+xml"), + ); + let kv = Arc::new(MemoryKv::default()); + + let output = sign_document( + &services(&http, &kv), + &format!("

x

{original}

y

"), + Surface::Page, + ); + + assert_eq!( + output, + Some(format!("

x

{signed_svg}

y

")), + "should place the signed serialization where the element was" + ); + let sent = &http.recorded_request_bodies()[0]; + assert!( + contains(sent, serialized.as_bytes()), + "should send the serialized tree" + ); + assert!( + !contains(sent, original.as_bytes()), + "should not send bytes the browser reads differently" + ); +} + +#[test] +fn unsigned_self_closing_svgs_never_add_whole_asset_signing_calls() { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + assert_eq!( + sign_document(&services(&http, &kv), "", Surface::Page), + None + ); + assert!(http.recorded_request_methods().is_empty()); + + let source = format!("data:image/png;base64,{}", STANDARD.encode(SOURCE)); + let signed = format!("data:image/png;base64,{}", STANDARD.encode(SIGNED)); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + assert_eq!( + sign_document( + &services(&http, &kv), + &format!(""), + Surface::Page, + ), + Some(format!("")) + ); + assert_eq!(http.recorded_request_methods(), ["POST"]); +} + +#[test] +fn original_svg_provenance_makes_no_child_or_whole_asset_signing_calls() { + let source = format!("data:image/png;base64,{}", STANDARD.encode(SOURCE)); + for provenance in [ + MANIFEST, + "", + "https://example.com/m", + "", + "", + ] { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + let document = format!( + "\ + {provenance}" + ); + assert_eq!( + sign_document_with_policies(&services(&http, &kv), &document, Surface::Page, Some(&[]),), + None, + "must preserve the entire original range for {provenance}" + ); + assert!(http.recorded_request_methods().is_empty(), "{provenance}"); + } +} + +#[test] +fn signing_an_outside_image_cannot_change_original_svg_provenance_ranges() { + let source = format!("data:image/png;base64,{}", STANDARD.encode(SOURCE)); + let signed = format!("data:image/png;base64,{}", STANDARD.encode(SIGNED)); + let svg = format!( + "{MANIFEST}\ + \ + " + ); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let document = format!("{svg}"); + assert_eq!( + sign_document_with_policies(&services(&http, &kv), &document, Surface::Page, Some(&[]),), + Some(format!("{svg}")), + "the rescan after outside edits must still preserve every SVG byte" + ); + assert_eq!(http.recorded_request_methods(), ["POST"]); +} + +#[test] +fn an_unsigned_svg_still_signs_its_attribute_css_and_then_the_whole_asset() { + let source = format!("data:image/png;base64,{}", STANDARD.encode(SOURCE)); + let signed = format!("data:image/png;base64,{}", STANDARD.encode(SIGNED)); + let root = ""; + let body = format!( + "Signed with C2PA\ + " + ); + let source_svg = format!("{root}{body}"); + let child_signed_svg = source_svg.replace(&source, &signed); + let signed_svg = format!("{root}{MANIFEST}{}", body.replace(&source, &signed)); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + http.push_response_with_headers( + 200, + signed_svg.as_bytes().to_vec(), + answer(signed_svg.as_bytes(), "image/svg+xml"), + ); + let kv = Arc::new(MemoryKv::default()); + assert_eq!( + sign_document_with_policies(&services(&http, &kv), &source_svg, Surface::Page, Some(&[]),), + Some(signed_svg) + ); + assert_eq!(http.recorded_request_methods(), ["POST", "POST"]); + let requests = http.recorded_request_bodies(); + assert!(contains(&requests[0], SOURCE)); + assert!(contains(&requests[1], child_signed_svg.as_bytes())); + assert!(!contains(&requests[1], source_svg.as_bytes())); +} + +#[test] +fn inline_svg_that_breaks_out_is_reported_without_a_call() { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + + let output = sign_document( + &services(&http, &kv), + "

x

", + Surface::Page, + ); + + assert_eq!(output, None); + assert!( + http.recorded_request_methods().is_empty(), + "should not call" + ); +} + +fn job<'a>( + reader_url: &'a str, + source: &'a Bytes, + sha256: &'a str, + reference: Option<&'a str>, + origin_host: Option<&'a str>, +) -> MediaJob<'a> { + MediaJob { + reader_url, + content_type: "image/png", + source, + source_sha256: sha256, + source_reference: reference, + source_origin_host: origin_host, + optimizer_output: false, + } +} + +fn deliver( + services: &RuntimeServices, + job: &MediaJob<'_>, +) -> Result { + let media = EncypherMedia::new(signer()); + block_on(media.sign(services, job)) +} + +const ORIGIN: Option<&str> = Some("origin.example.com"); + +#[test] +fn delivered_media_outside_the_claim_with_other_bytes_or_no_origin_host_is_refused() { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let source = Bytes::from_static(SOURCE); + let sha = sha256_hex(SOURCE); + let reader = "https://publisher.example.com/a.png"; + + let outside = deliver( + &services, + &job("https://cdn.example.org/a.png", &source, &sha, None, ORIGIN), + ) + .expect_err("should refuse another host"); + assert_eq!(outside.reason, "outside_claim"); + let other = "0".repeat(64); + let mismatch = deliver(&services, &job(reader, &source, &other, None, ORIGIN)) + .expect_err("should refuse other bytes"); + assert_eq!(mismatch.reason, "source_digest_mismatch"); + for host in [ + None, + Some(""), + Some("origin.example.com:8443"), + Some("user@origin.example.com"), + Some("origin.example.com/images"), + Some("Origin.example.com"), + ] { + let refused = deliver(&services, &job(reader, &source, &sha, None, host)) + .expect_err("should refuse a missing or malformed origin host"); + assert_eq!(refused.reason, "invalid_origin_host", "{host:?}"); + } + assert!( + http.recorded_request_methods().is_empty(), + "should not call" + ); +} + +#[test] +fn delivered_media_carries_a_link_for_a_remote_manifest_and_only_a_safe_reference() { + for (reference, origin_host, expected_reference, expected_origin) in [ + ( + Some("https://publisher.example.com/images/a.png"), + "publisher.example.com", + "\"reference\":\"https://publisher.example.com/images/a.png\"", + "\"origin_host\":\"publisher.example.com\"", + ), + ( + Some("https://origin.example.com/images/a.png"), + "origin.example.com", + "\"reference\":null", + "\"origin_host\":\"origin.example.com\"", + ), + // A fetched address with a query is no reference, but its host is + // still the origin's. + ( + None, + "origin.example.com", + "\"reference\":null", + "\"origin_host\":\"origin.example.com\"", + ), + ] { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&json!({"policy_digest": POLICY_DIGEST})).expect("JSON"), + ); + http.push_response_with_headers(200, SOURCE.to_vec(), external_answer(SOURCE)); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let source = Bytes::from_static(SOURCE); + let sha = sha256_hex(SOURCE); + + let delivered = deliver( + &services, + &job( + "https://publisher.example.com/images/a.png", + &source, + &sha, + reference, + Some(origin_host), + ), + ) + .expect("should sign"); + + assert_eq!( + delivered.bytes.as_ref(), + SOURCE, + "should deliver the answer's bytes" + ); + assert_eq!(delivered.content_type, "image/png"); + assert_eq!( + delivered.namespace, + format!( + "{NAMESPACE_VERSION}:{POLICY_DIGEST}:{}", + signer().cache_context + ), + "should name the policy it was signed under for core's kept copy" + ); + let named = |name: &str| { + delivered + .headers + .iter() + .find(|(header, _)| header.as_str() == name) + .map(|(_, value)| value.to_str().expect("text").to_owned()) + }; + assert_eq!( + named("link").as_deref(), + Some("; rel=\"c2pa-manifest\""), + "should name the remote manifest on every response" + ); + assert_eq!(named("x-encypher-document-id").as_deref(), Some("doc-1")); + let body = &http.recorded_request_bodies()[1]; + assert!( + contains(body, expected_reference.as_bytes()), + "should send {expected_reference}" + ); + assert!( + contains(body, expected_origin.as_bytes()), + "should send {expected_origin}" + ); + assert!( + contains(body, b"\"http_link_header\":true"), + "should say a Link can be delivered" + ); + } +} + +#[test] +fn an_optimizer_answer_is_sent_requiring_its_manifest_and_bound_by_its_bytes() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&json!({"policy_digest": POLICY_DIGEST})).expect("JSON"), + ); + http.push_response_with_headers(200, SOURCE.to_vec(), external_answer(SOURCE)); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let source = Bytes::from_static(SOURCE); + let sha = sha256_hex(SOURCE); + let optimized = MediaJob { + optimizer_output: true, + ..job( + "https://publisher.example.com/images/a.png", + &source, + &sha, + Some("https://publisher.example.com/images/a.png"), + Some("publisher.example.com"), + ) + }; + + deliver(&services, &optimized).expect("should sign"); + + let body = &http.recorded_request_bodies()[1]; + assert!( + contains(body, b"\"require_embedded_manifest\":true"), + "should require the optimizer's embedded manifest" + ); + assert!( + contains(body, b"\"reference\":null"), + "should not name the address the optimizer read as the source" + ); + assert!( + contains(body, b"\"operations\":[]"), + "should report no operation of the edge's own" + ); +} + +#[test] +fn a_result_is_never_reused_for_the_same_bytes_from_another_origin() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + let kv = Arc::new(MemoryKv::default()); + let services = services(&http, &kv); + let signer = signer(); + let sha = sha256_hex(SOURCE); + + for host in ["origin-a.example.com", "origin-b.example.com"] { + let source = MediaSource { + origin_host: Some(host), + ..png_source(&sha) + }; + let (_, reason) = sign(&signer, &services, &source).expect("should sign"); + assert_eq!(reason, Reason::Signed, "{host}: should ask the endpoint"); + } + assert_eq!( + http.recorded_request_methods().len(), + 2, + "should bind each result to its origin" + ); + let headers = http.recorded_request_headers(); + assert_ne!( + header(&headers[0], "idempotency-key"), + header(&headers[1], "idempotency-key"), + "should replay neither under the other's key" + ); +} + +#[test] +fn media_is_recognized_by_its_type_on_every_path_but_excluded_prefixes() { + let media = EncypherMedia::new(signer()); + for path in [ + "/images/a.png", + "/video/clip.MOV", + "/audio/a.mp3", + "/img/123", + "/media", + ] { + assert!(media.handles_path(path), "should consider {path}"); + } + for path in [ + "/api/v1/a.png", + "/wp-admin/x.jpg", + "/.well-known/a.png", + "/feed/", + ] { + assert!(!media.handles_path(path), "should exclude {path}"); + } + for media_type in [ + "image/webp", + "image/svg+xml", + "video/mp4", + "video/webm", + "audio/mpeg", + "font/woff2", + ] { + assert!( + media.handles_media_type(media_type), + "should sign {media_type}" + ); + } + for media_type in [ + "image/", + "text/css", + "text/html", + "application/pdf", + "video/iso.segment", + ] { + assert!( + !media.handles_media_type(media_type), + "should not sign {media_type}" + ); + } + assert_eq!(media.max_source_bytes(), 5_000_000, "should bound captures"); +} + +#[test] +fn the_kept_copy_namespace_fails_closed_on_revocation_however_warm_the_caches() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&json!({"policy_digest": POLICY_DIGEST})).expect("JSON"), + ); + http.push_response(403, b"{}".to_vec()); + let kv = Arc::new(MemoryKv::default()); + let runtime = services(&http, &kv); + let media = EncypherMedia::new(signer()); + // The article path's policy cache holds a fresh digest throughout. + let signer = signer(); + kv.set( + &signer.policy_key, + Bytes::from( + serde_json::to_vec(&json!({ + "kind": "digest", + "policy_digest": POLICY_DIGEST, + "expires_at": unix_now() + 300, + })) + .expect("JSON"), + ), + ); + + let namespace = block_on(media.namespace(&runtime)).expect("should answer a namespace"); + assert!( + namespace.contains(POLICY_DIGEST), + "should bind the policy digest: {namespace}" + ); + assert!( + !namespace.contains(TOKEN), + "should never carry the credential" + ); + let failure = block_on(media.namespace(&runtime)) + .expect_err("should refuse a revoked credential with the cached digest still fresh"); + assert_eq!(failure.reason, "forbidden"); +} + +#[test] +fn a_warm_signed_result_is_not_delivered_under_a_revoked_credential() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&json!({"policy_digest": POLICY_DIGEST})).expect("JSON"), + ); + http.push_response_with_headers(200, SIGNED.to_vec(), answer(SIGNED, "image/png")); + http.push_response(403, b"{}".to_vec()); + let kv = Arc::new(MemoryKv::default()); + let runtime = services(&http, &kv); + let source = Bytes::from_static(SOURCE); + let sha = sha256_hex(SOURCE); + let job = job( + "https://publisher.example.com/images/a.png", + &source, + &sha, + None, + ORIGIN, + ); + deliver(&runtime, &job).expect("should sign and cache the result"); + assert!( + !kv.keys_with(MEDIA_KEY_PREFIX).is_empty(), + "should hold the signed result" + ); + + let refused = deliver(&runtime, &job).expect_err("should refuse the revoked credential"); + assert_eq!(refused.reason, "forbidden"); + assert_eq!( + http.recorded_request_methods(), + ["GET", "POST", "GET"], + "should neither sign again nor deliver the cached result" + ); +} diff --git a/crates/provenance/encypher/src/processor/prepared.rs b/crates/provenance/encypher/src/processor/prepared.rs new file mode 100644 index 0000000000..8aa58ad3ad --- /dev/null +++ b/crates/provenance/encypher/src/processor/prepared.rs @@ -0,0 +1,1087 @@ +//! Large media signed where it is delivered, through the prepared lane. +//! +//! A source too large to capture is never uploaded and never held whole. +//! Core pins one version of it by its strong `ETag` and hashes it in bounded +//! chunks. This lane then asks the Encypher prepared-signing endpoints for a +//! remote manifest over those exact bytes: +//! +//! 1. The policy, then the lane's capabilities for the asset. +//! 2. The signed probe release the service advertises, verified against the +//! pinned release key before it runs. +//! 3. The probe's disclosure of the source's container structure, read from +//! the pinned version. A source that already carries an embedded C2PA +//! manifest stops here: a remote manifest cannot stand beside it, as an +//! embedded manifest wins over a `Link`, so the source is delivered as it +//! is, with its own credentials, reported as existing provenance kept and +//! never as newly signed or verified. +//! 4. `prepare`, then the probe's check that the server's plan covers the +//! whole source unchanged, then a last check that the version is still +//! current, then `finalize`. +//! 5. The returned store is checked to bind the whole-source digest. The +//! reader gets the origin's bytes with a `Link` to that store. +//! +//! Every failure is a refusal with a code. Core then delivers the origin's +//! bytes unsigned; nothing is published for a source the lane did not +//! verify. + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use http::{HeaderValue, Method, Request, StatusCode, header}; +use serde::Serialize; +use serde_json::{Value, json}; +use std::time::Duration; +use trusted_server_core::media_signing::{ + LargeMediaJob, LargeSignedMedia, MediaSourceError, VersionedMediaSource, +}; +use trusted_server_core::middleware::ResponseHeader; +use trusted_server_core::platform::RuntimeServices; + +use super::{ + Deadline, Reason, SKIP_WRITE_RESERVE, Signer, WORKER_TOKEN_HEADER, is_excluded_prefix, +}; +use crate::api::valid_policy_digest; +use crate::media_api::{ + self, IDEMPOTENCY_KEY_HEADER, canonical_media_type, safe_source_reference, sha256_hex, + valid_origin_host, +}; +use crate::prepared_probe::{ + ProbeCallError, ProbeDescriptor, ProbeLimits, ProbeModule, ProbeSource, ReleaseAuthority, +}; + +const CAPABILITIES_PATH: &str = "/api/v1/public/cdn/media/hash/capabilities"; +const PREPARE_PATH: &str = "/api/v1/public/cdn/media/hash/prepare"; +const FINALIZE_PATH: &str = "/api/v1/public/cdn/media/hash/finalize"; +/// Signed probe releases, served without credentials and never changed. +const PROBE_PATH: &str = "/api/v1/"; +const PROTOCOL: &str = "encypher-prepared-signing/2"; +/// The cached artifact of a probe release, by its SHA-256. +const PROBE_KEY_PREFIX: &str = "encypher:probe:v1"; +/// The largest probe artifact accepted. +const MAX_PROBE_BYTES: usize = 4 * 1024 * 1024; +/// A capabilities answer is a few descriptors and limits. +const MAX_CAPABILITIES_BYTES: usize = 64 * 1024; +/// A prepare answer is ids and the planner's geometry. +const MAX_PREPARE_BYTES: usize = 1024 * 1024; +/// A finalize answer carries the manifest store, base64 encoded, besides +/// ids and URLs. +const FINALIZE_OVERHEAD_BYTES: usize = 64 * 1024; +/// The largest finalized manifest store the lane takes, whatever the +/// service allows: the answer, its decoded store and the probe's copy of it +/// must fit an edge instance. An external manifest of an unsigned source, +/// with no ingredient chain or identity assertion, is far smaller. +const MAX_FINALIZED_MANIFEST_BYTES: usize = 2 * 1024 * 1024; +/// The refusal for a finalized manifest larger than the lane takes. +const MANIFEST_TOO_LARGE: &str = "manifest_too_large"; +/// The outcome of a source that already carries an embedded C2PA manifest: +/// delivered as it is, its own provenance kept, nothing newly signed. +const EMBEDDED_PROVENANCE: &str = "source_has_embedded_provenance"; +/// The outcome of a source whose XMP already names a C2PA manifest: delivered +/// as it is, its own provenance kept, nothing newly signed. +const REFERENCED_PROVENANCE: &str = "source_has_provenance_reference"; + +const INSTANCE_ID_HEADER: &str = "x-encypher-manifest-instance-id"; +const DOCUMENT_ID_HEADER: &str = "x-encypher-document-id"; +const VERIFICATION_URL_HEADER: &str = "x-encypher-verification-url"; + +/// Why the prepared lane did not sign a large source. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum LargeRefusal { + /// A refusal the module shares with its other lanes. + Module(Reason), + /// The pinned source could not be read as the version it was pinned at. + Source(MediaSourceError), + /// A refusal particular to this lane. + Lane(&'static str), +} + +impl LargeRefusal { + pub(super) fn code(self) -> &'static str { + match self { + Self::Module(reason) => reason.code(), + Self::Source(error) => error.code(), + Self::Lane(code) => code, + } + } + + /// Whether the source was delivered with its own embedded credentials, + /// its provenance kept rather than lost to a failure. + pub(super) fn retains_provenance(self) -> bool { + self == Self::Lane(EMBEDDED_PROVENANCE) || self == Self::Lane(REFERENCED_PROVENANCE) + } +} + +impl From for LargeRefusal { + fn from(reason: Reason) -> Self { + Self::Module(reason) + } +} + +impl From for LargeRefusal { + fn from(error: MediaSourceError) -> Self { + Self::Source(error) + } +} + +/// The pinned version, as the probe reads it. +struct PinnedSource<'a>(&'a VersionedMediaSource<'a>); + +impl ProbeSource for PinnedSource<'_> { + type Error = MediaSourceError; + + fn size(&self) -> u64 { + self.0.size() + } + + async fn read_exact_at( + &self, + offset: u64, + destination: &mut [u8], + ) -> Result<(), MediaSourceError> { + self.0.read_exact_at(offset, destination).await + } +} + +/// The lane's capabilities for one asset, as the service answers them. +struct Capabilities { + policy_digest: String, + formats: Vec, + total_size_bytes: u64, + manifest_decoded_bytes: usize, + probes: Vec, +} + +impl Capabilities { + /// The largest finalized store the lane takes: the service's maximum, + /// never more than the edge's own. + fn manifest_limit(&self) -> usize { + self.manifest_decoded_bytes + .min(MAX_FINALIZED_MANIFEST_BYTES) + } + + fn parse(body: &[u8]) -> Option { + let value: Value = serde_json::from_slice(body).ok()?; + if value.get("protocol")?.as_str()? != PROTOCOL || !value.get("enabled")?.as_bool()? { + return None; + } + let maximums = value.get("maximums")?; + Some(Self { + policy_digest: value.get("policy_digest")?.as_str()?.to_owned(), + formats: value + .get("formats")? + .as_array()? + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + total_size_bytes: maximums.get("total_size_bytes")?.as_u64()?, + manifest_decoded_bytes: usize::try_from( + maximums.get("manifest_decoded_bytes")?.as_u64()?, + ) + .ok()?, + probes: ProbeDescriptor::parse_list(value.get("probe")), + }) + } +} + +/// What the lane signs one source with, once the gates passed. +#[derive(Clone, Copy)] +struct Lane<'a> { + job: &'a LargeMediaJob<'a>, + reference: &'a str, + mime: &'a str, + policy_digest: &'a str, + /// The release key probes are held to: production, except in tests. + authority: &'a ReleaseAuthority, +} + +/// The prepare request, serialized from borrowed values so the disclosure +/// is not copied. +#[derive(Serialize)] +struct PrepareBody<'a> { + page_url: &'a str, + expected_policy_digest: &'a str, + protocol: &'a str, + probe: ProbeSelection<'a>, + source: PreparedSource<'a>, + structure: Vec>, +} + +#[derive(Serialize)] +struct ProbeSelection<'a> { + sha256: &'a str, + planner_id: &'a str, +} + +#[derive(Serialize)] +struct PreparedSource<'a> { + reference: &'a str, + origin_host: &'a str, + mime: &'a str, + sha256: &'a str, + byte_length: u64, +} + +#[derive(Serialize)] +struct StructureChunk<'a> { + offset: u64, + b64: &'a str, +} + +/// The verified finalize answer: what the delivered headers name. +struct Finalized { + manifest_url: String, + instance_id: String, + document_id: String, + verification_url: String, +} + +/// Whether `value` is an identifier safe in a response header. +fn valid_identifier(value: &str) -> bool { + !value.is_empty() && value.len() <= 256 && value.bytes().all(|byte| byte.is_ascii_graphic()) +} + +/// Whether `value` is a canonical absolute `https` URL safe inside a `Link`. +fn valid_https_url(value: &str) -> bool { + value.len() <= 2048 + && value + .bytes() + .all(|byte| byte.is_ascii_graphic() && !matches!(byte, b'<' | b'>' | b'"')) + && url::Url::parse(value).is_ok_and(|parsed| { + parsed.scheme() == "https" && parsed.host_str().is_some() && parsed.as_str() == value + }) +} + +/// The reason a probe call failed, logged with the probe's own code. +fn probe_refusal>(op: &str, error: ProbeCallError) -> LargeRefusal { + log::info!("encypher: large media probe op={op} code={}", error.code()); + match error { + ProbeCallError::Source(error) => error.into(), + ProbeCallError::Refused(refusal) if refusal.integrity => { + LargeRefusal::Lane("source_provenance_invalid") + } + ProbeCallError::Refused(_) => LargeRefusal::Lane("probe_refused"), + ProbeCallError::Failed(_) => LargeRefusal::Lane("probe_failed"), + } +} + +/// Whether `items` is the one program of an unchanged external plan: the +/// whole source, once, with nothing excluded or inserted. +fn covers_whole_source(items: &[Value], size: u64) -> bool { + matches!(items, [item] if item.get("range").and_then(Value::as_array).is_some_and(|range| { + range.len() == 2 + && range.first().and_then(Value::as_u64) == Some(0) + && range.get(1).and_then(Value::as_u64) == Some(size) + }) && item.as_object().is_some_and(|object| object.len() == 1)) +} + +/// The reason for an error answer of the prepared endpoints, by status and +/// failure code. +fn answer_refusal(status: StatusCode, code: Option<&str>) -> LargeRefusal { + match (status.as_u16(), code) { + (409, Some("TRUSTED_SERVER_POLICY_CHANGED")) => Reason::PolicyChanged.into(), + (409, Some("PROBE_VERSION_UNSUPPORTED" | "PLAN_STALE")) => { + LargeRefusal::Lane("probe_release_stale") + } + (409, Some("SOURCE_DIGEST_MISMATCH")) => LargeRefusal::Lane("source_digest_mismatch"), + (422, Some("SOURCE_PROVENANCE_INVALID")) => LargeRefusal::Lane("source_provenance_invalid"), + // A store the probe's disclosure did not report: kept, not signed. + (422, Some("SOURCE_MANIFEST_PRESENT")) => LargeRefusal::Lane(EMBEDDED_PROVENANCE), + (422, Some("HASH_MODE_UNSUPPORTED")) => LargeRefusal::Lane("format_unsupported"), + (403 | 404, _) => Reason::Forbidden.into(), + (409, _) => LargeRefusal::Lane("prepared_conflict"), + (413, _) => Reason::PayloadTooLarge.into(), + (422, _) => Reason::Unprocessable.into(), + (429, _) => Reason::RateLimited.into(), + (500..=599, _) => Reason::ServerError.into(), + _ => Reason::UnexpectedStatus.into(), + } +} + +impl Signer { + /// Largest source the prepared lane signs; larger media is delivered + /// unsigned. + pub(super) fn max_large_media_bytes(&self) -> u64 { + self.config.max_large_media_bytes + } + + /// The whole time core and this lane may spend on one large source, or + /// none when the lane is off. + pub(super) fn max_large_media_duration(&self) -> Duration { + if self.config.max_large_media_bytes == 0 { + Duration::ZERO + } else { + Duration::from_millis(self.config.max_large_media_ms) + } + } + + /// Sign one large source core pinned and hashed, returning the headers + /// every response of it carries and the namespace it was signed under. + /// Probe releases are always held to the pinned production key. + pub(super) async fn sign_large( + &self, + services: &RuntimeServices, + job: &LargeMediaJob<'_>, + ) -> Result { + self.sign_large_with(services, job, &ReleaseAuthority::production()) + .await + } + + /// [`Self::sign_large`] with the release `authority` given, which only + /// tests set to anything but production. + async fn sign_large_with( + &self, + services: &RuntimeServices, + job: &LargeMediaJob<'_>, + authority: &ReleaseAuthority, + ) -> Result { + let source = job.source; + let size = source.size(); + let reader_url = url::Url::parse(job.reader_url).map_err(|_| Reason::OutsideClaim)?; + let reader_host = reader_url.host_str().ok_or(Reason::OutsideClaim)?; + if !matches!(reader_url.scheme(), "https" | "http") + || !self.claims_host(reader_host) + || is_excluded_prefix(reader_url.path()) + { + return Err(Reason::OutsideClaim.into()); + } + if size == 0 || size > self.max_large_media_bytes() { + return Err(Reason::MediaTooLarge.into()); + } + if !valid_origin_host(job.source_origin_host) { + return Err(Reason::InvalidOriginHost.into()); + } + if !valid_policy_digest(job.source_sha256) { + return Err(Reason::SourceDigestMismatch.into()); + } + // The fetched address when it names the origin's bytes; otherwise + // the reader's own address, which names the same bytes here because + // they are delivered unchanged. A source with neither is refused. + let reference = job + .source_reference + .filter(|reference| safe_source_reference(reference, job.source_origin_host)) + .or_else(|| { + job.reader_reference.filter(|reference| { + *reference == job.reader_url && safe_source_reference(reference, reader_host) + }) + }) + .ok_or(LargeRefusal::Lane("source_reference_required"))?; + let mime = canonical_media_type(job.content_type); + + // What is left of the lane's budget after core hashed the source, + // less the reserve a policy eviction may still use. + let mut work = Deadline::new( + services.timer(), + job.remaining.saturating_sub(SKIP_WRITE_RESERVE), + ); + work.start().await; + // Asked live: the namespace core keeps the result under must be the + // policy the service holds now, never a cached one a revocation + // outlived. + let policy_digest = self + .request_policy(job.reader_url, services, &mut work) + .await?; + let lane = Lane { + job, + reference, + mime: &mime, + policy_digest: &policy_digest, + authority, + }; + let result = self.run_prepared(services, &lane, &mut work).await; + if let Err(LargeRefusal::Module(Reason::PolicyChanged | Reason::PolicyDigestMismatch)) = + &result + { + self.evict_policy(services.kv_store(), &mut Self::skip_deadline(services)) + .await; + } + let finalized = result?; + let link = format!("<{}>; rel=\"c2pa-manifest\"", finalized.manifest_url); + let mut headers: Vec = Vec::with_capacity(4); + for (name, value) in [ + (header::LINK, link.as_str()), + ( + http::HeaderName::from_static(INSTANCE_ID_HEADER), + finalized.instance_id.as_str(), + ), + ( + http::HeaderName::from_static(DOCUMENT_ID_HEADER), + finalized.document_id.as_str(), + ), + ( + http::HeaderName::from_static(VERIFICATION_URL_HEADER), + finalized.verification_url.as_str(), + ), + ] { + headers.push(( + name, + HeaderValue::from_str(value).map_err(|_| Reason::InvalidSignedResponse)?, + )); + } + Ok(LargeSignedMedia { + headers, + namespace: self.kept_namespace(&policy_digest), + }) + } + + /// The lane's steps after the gates: capabilities, probe, disclosure, + /// prepare, plan check, version check, finalize, store check. + async fn run_prepared( + &self, + services: &RuntimeServices, + lane: &Lane<'_>, + work: &mut Deadline<'_>, + ) -> Result { + let Lane { + job, + reference, + mime, + policy_digest, + authority, + } = *lane; + let source = job.source; + let size = source.size(); + let capabilities = self.capabilities(services, job.reader_url, work).await?; + if capabilities.policy_digest != policy_digest { + return Err(Reason::PolicyChanged.into()); + } + if !capabilities.formats.iter().any(|format| format == mime) { + return Err(LargeRefusal::Lane("format_unsupported")); + } + if size > capabilities.total_size_bytes { + return Err(Reason::MediaTooLarge.into()); + } + let probe = self + .load_probe(services, authority, &capabilities.probes, work) + .await?; + let pinned = PinnedSource(source); + + let mut extracted = match work + .race(probe.call( + "extract", + &json!({"mime": mime, "total_size": size, "disposition": "external"}), + &pinned, + )) + .await + { + None => return Err(Reason::Timeout.into()), + Some(result) => result.map_err(|error| probe_refusal("extract", error))?, + }; + // An embedded manifest wins over a `Link`, so a remote one would be + // ignored beside it, and a source whose XMP already names a manifest + // has provenance of its own: either way the source keeps it. A + // release that does not say whether the XMP names one cannot be + // trusted to have looked. + if extracted.get("credentials").is_some() { + return Err(LargeRefusal::Lane(EMBEDDED_PROVENANCE)); + } + match extracted + .get("has_provenance_reference") + .and_then(Value::as_bool) + { + Some(true) => return Err(LargeRefusal::Lane(REFERENCED_PROVENANCE)), + Some(false) => {} + None => return Err(LargeRefusal::Lane("probe_result_invalid")), + } + // The disclosure is taken out of the result, not copied: it is sent + // at prepare and handed back to the probe at plan verification. + let disclosure = extracted + .get_mut("chunks") + .map(Value::take) + .ok_or(LargeRefusal::Lane("probe_result_invalid"))?; + drop(extracted); + let structure = disclosure + .as_array() + .ok_or(LargeRefusal::Lane("probe_result_invalid"))? + .iter() + .map(|chunk| { + Some(StructureChunk { + offset: chunk.get("offset")?.as_u64()?, + b64: chunk.get("data")?.as_str()?, + }) + }) + .collect::>>() + .ok_or(LargeRefusal::Lane("probe_result_invalid"))?; + + let idempotency_key = sha256_hex( + format!( + "prepared\n{}\n{policy_digest}\n{}\n{reference}\n{}\n{mime}\n{}\n{size}\n{}", + self.cache_context, + job.reader_url, + job.source_origin_host, + job.source_sha256, + source.version().etag(), + ) + .as_bytes(), + ); + let descriptor = probe.descriptor(); + let prepare = PrepareBody { + page_url: job.reader_url, + expected_policy_digest: policy_digest, + protocol: PROTOCOL, + probe: ProbeSelection { + sha256: &descriptor.sha256, + planner_id: &descriptor.planner_id, + }, + source: PreparedSource { + reference, + origin_host: job.source_origin_host, + mime, + sha256: job.source_sha256, + byte_length: size, + }, + structure, + }; + let mut prepared = self + .post( + services, + PREPARE_PATH, + &prepare, + &idempotency_key, + MAX_PREPARE_BYTES, + work, + ) + .await?; + drop(prepare); + let plan_id = prepared + .get("plan_id") + .and_then(Value::as_str) + .filter(|id| id.len() == 36) + .ok_or(LargeRefusal::Lane("prepare_answer_invalid"))? + .to_owned(); + if prepared.get("policy_digest").and_then(Value::as_str) != Some(policy_digest) { + return Err(Reason::PolicyDigestMismatch.into()); + } + let unchanged = prepared + .get("local_ops") + .and_then(Value::as_array) + .is_some_and(Vec::is_empty) + && prepared.get("effective_action").and_then(Value::as_str) == Some("c2pa.opened"); + let geometry = prepared + .get_mut("geometry") + .map(Value::take) + .filter(|geometry| { + geometry.get("disposition").and_then(Value::as_str) == Some("external") + }); + drop(prepared); + let Some(geometry) = geometry.filter(|_| unchanged) else { + return Err(LargeRefusal::Lane("prepare_answer_invalid")); + }; + + // The probe recomputes the server's plan from the same bytes; an + // unchanged external plan hashes the whole source once. + let verified = match work + .race(probe.call( + "verify_plan", + &json!({ + "mime": mime, + "total_size": size, + "disclosure": disclosure, + "geometry": geometry, + }), + &pinned, + )) + .await + { + None => return Err(Reason::Timeout.into()), + Some(result) => result.map_err(|error| probe_refusal("verify_plan", error))?, + }; + let items = verified + .get("hash_items") + .and_then(Value::as_array) + .ok_or(LargeRefusal::Lane("probe_result_invalid"))?; + if !covers_whole_source(items, size) { + return Err(LargeRefusal::Lane("plan_not_whole_source")); + } + drop(verified); + match work.race(source.confirm_current()).await { + None => return Err(Reason::Timeout.into()), + Some(result) => result?, + } + + let finalize = json!({ + "page_url": job.reader_url, + "expected_policy_digest": policy_digest, + "protocol": PROTOCOL, + "plan_id": plan_id, + "binding": {"type": "data_hash", "digest": job.source_sha256}, + "xmp_reference": false, + }); + let manifest_limit = capabilities.manifest_limit(); + let answer_limit = manifest_limit + .saturating_mul(4) + .div_ceil(3) + .saturating_add(FINALIZE_OVERHEAD_BYTES); + let finalized = self + .post( + services, + FINALIZE_PATH, + &finalize, + &idempotency_key, + answer_limit, + work, + ) + .await + .map_err(|refusal| match refusal { + LargeRefusal::Module(Reason::ResponseTooLarge) => { + LargeRefusal::Lane(MANIFEST_TOO_LARGE) + } + other => other, + })?; + self.verify_finalized(&probe, &finalized, job, policy_digest, &capabilities, work) + .await + } + + /// Checks the finalize answer against the plan and the pinned source, + /// and that the store it returns binds the whole-source digest. + async fn verify_finalized( + &self, + probe: &ProbeModule, + finalized: &Value, + job: &LargeMediaJob<'_>, + policy_digest: &str, + capabilities: &Capabilities, + work: &mut Deadline<'_>, + ) -> Result { + let text = |name: &str| finalized.get(name).and_then(Value::as_str); + if text("policy_digest") != Some(policy_digest) { + return Err(Reason::PolicyDigestMismatch.into()); + } + let external = text("manifest_disposition") == Some("external") + && text("xmp_reference") == Some("declined") + && text("binding_algorithm") == Some("c2pa.hash.data") + && finalized.get("expected_final_size").and_then(Value::as_u64) + == Some(job.source.size()); + if !external { + return Err(LargeRefusal::Lane("finalize_answer_invalid")); + } + let store_b64 = text("manifest_store_b64").ok_or(Reason::UnsignedResponse)?; + let manifest_limit = capabilities.manifest_limit(); + if store_b64.len() + > manifest_limit + .saturating_mul(4) + .div_ceil(3) + .saturating_add(3) + { + return Err(LargeRefusal::Lane(MANIFEST_TOO_LARGE)); + } + let store = STANDARD + .decode(store_b64) + .map_err(|_| Reason::InvalidSignedResponse)?; + if store.is_empty() { + return Err(Reason::InvalidSignedResponse.into()); + } + if store.len() > manifest_limit { + return Err(LargeRefusal::Lane(MANIFEST_TOO_LARGE)); + } + if text("manifest_sha256") != Some(sha256_hex(&store).as_str()) { + return Err(Reason::SignedDigestMismatch.into()); + } + let manifest_url = text("manifest_url") + .filter(|url| valid_https_url(url)) + .ok_or(Reason::InvalidSignedResponse)?; + let verification_url = text("verification_url") + .filter(|url| valid_https_url(url)) + .ok_or(Reason::InvalidSignedResponse)?; + let instance_id = text("instance_id") + .filter(|id| valid_identifier(id)) + .ok_or(Reason::InvalidSignedResponse)?; + let document_id = text("document_id") + .filter(|id| valid_identifier(id)) + .ok_or(Reason::InvalidSignedResponse)?; + // The store's hard binding must be over the whole source, nothing + // excluded, with the digest core computed over the pinned version. + let checked = match work + .race(probe.call_without_source::( + "check_store", + &json!({ + "manifest_store_b64": store_b64, + "digest": job.source_sha256, + "binding_recipe": {"type": "data_hash", "algorithm": "sha256", "exclusions": []}, + }), + )) + .await + { + None => return Err(Reason::Timeout.into()), + Some(result) => result.map_err(|error| probe_refusal("check_store", error))?, + }; + if checked.get("ok").and_then(Value::as_bool) != Some(true) { + return Err(Reason::SignedDigestMismatch.into()); + } + Ok(Finalized { + manifest_url: manifest_url.to_owned(), + instance_id: instance_id.to_owned(), + document_id: document_id.to_owned(), + verification_url: verification_url.to_owned(), + }) + } + + /// The lane's capabilities for the asset at `page_url`. + async fn capabilities( + &self, + services: &RuntimeServices, + page_url: &str, + work: &mut Deadline<'_>, + ) -> Result { + let mut url = url::Url::parse(&format!( + "{}{CAPABILITIES_PATH}", + self.config.api_base_url.trim_end_matches('/') + )) + .map_err(|_| Reason::RequestBuild)?; + url.query_pairs_mut().append_pair("page_url", page_url); + let request = Request::builder() + .method(Method::GET) + .uri(url.as_str()) + .header(WORKER_TOKEN_HEADER, self.worker_token()?) + .body(EdgeBody::empty()) + .map_err(|_| Reason::RequestBuild)?; + let body = self + .exchange(services, request, MAX_CAPABILITIES_BYTES, work) + .await?; + Capabilities::parse(&body).ok_or(LargeRefusal::Lane("capabilities_invalid")) + } + + /// The newest verified release the service advertises, from the cache or + /// fetched, compiled with the edge's limits. + async fn load_probe( + &self, + services: &RuntimeServices, + authority: &ReleaseAuthority, + advertised: &[ProbeDescriptor], + work: &mut Deadline<'_>, + ) -> Result { + let descriptor = authority.select(advertised).map_err(|error| { + log::info!("encypher: large media probe release code={}", error.code()); + LargeRefusal::Lane(error.code()) + })?; + let kv = services.kv_store(); + let key = format!("{PROBE_KEY_PREFIX}:{}", descriptor.sha256); + let cached = match work.race(kv.get_bytes(&key)).await { + Some(Ok(Some(bytes))) if sha256_hex(&bytes) == descriptor.sha256 => Some(bytes), + _ => None, + }; + let artifact = match cached { + Some(bytes) => bytes, + None => { + let request = Request::builder() + .method(Method::GET) + .uri(format!( + "{}{PROBE_PATH}{}", + self.config.api_base_url.trim_end_matches('/'), + descriptor.artifact_path() + )) + .body(EdgeBody::empty()) + .map_err(|_| Reason::RequestBuild)?; + let bytes = self + .exchange(services, request, MAX_PROBE_BYTES, work) + .await?; + if sha256_hex(&bytes) != descriptor.sha256 { + return Err(LargeRefusal::Lane("PROBE_RELEASE_INVALID")); + } + // Releases never change, so a cached copy needs no expiry + // beyond the plan lifetime. + let ttl = Duration::from_secs(self.config.plan_ttl_seconds); + if !matches!( + work.race(kv.put_bytes_with_ttl(&key, bytes.clone(), ttl)) + .await, + Some(Ok(())) + ) { + log::debug!("encypher: probe release cache write failed"); + } + bytes + } + }; + let loaded = match work + // The host's default limits are the edge's: structure only, never + // the payload. A release that runs out of fuel is refused. + .race(ProbeModule::load( + authority, + descriptor, + &artifact, + ProbeLimits::default(), + )) + .await + { + None => return Err(Reason::Timeout.into()), + Some(result) => result, + }; + loaded.map_err(|error| { + log::info!("encypher: large media probe release code={}", error.code()); + LargeRefusal::Lane(error.code()) + }) + } + + /// One authenticated JSON `POST` to a prepared endpoint. + async fn post( + &self, + services: &RuntimeServices, + path: &str, + body: &impl Serialize, + idempotency_key: &str, + max_response_bytes: usize, + work: &mut Deadline<'_>, + ) -> Result { + let body = serde_json::to_vec(body).map_err(|_| Reason::RequestBuild)?; + let request = Request::builder() + .method(Method::POST) + .uri(format!( + "{}{path}", + self.config.api_base_url.trim_end_matches('/') + )) + .header(header::CONTENT_TYPE, "application/json") + .header(WORKER_TOKEN_HEADER, self.worker_token()?) + .header(IDEMPOTENCY_KEY_HEADER, idempotency_key) + .body(EdgeBody::from(body)) + .map_err(|_| Reason::RequestBuild)?; + let answer = self + .exchange(services, request, max_response_bytes, work) + .await?; + serde_json::from_slice(&answer).map_err(|_| Reason::MalformedResponse.into()) + } + + /// One bounded call; a non-`200` answer is a refusal with its code. + async fn exchange( + &self, + services: &RuntimeServices, + request: Request, + max_response_bytes: usize, + work: &mut Deadline<'_>, + ) -> Result { + let limit = max_response_bytes.saturating_add(1); + let response = self.send_api(services, request, limit, work).await?; + let status = response.status(); + if status != StatusCode::OK { + let code = work + .race( + response + .into_body() + .into_bytes_bounded(media_api::MAX_ERROR_BODY_BYTES), + ) + .await + .and_then(Result::ok) + .and_then(|body| media_api::error_code(&body)); + log::info!( + "encypher: large media answer status={} code={}", + status.as_u16(), + code.as_deref().unwrap_or("none") + ); + return Err(answer_refusal(status, code.as_deref())); + } + let body = match work + .race(response.into_body().into_bytes_bounded(limit)) + .await + { + None => return Err(Reason::Timeout.into()), + Some(Err(_)) => return Err(Reason::ResponseTooLarge.into()), + Some(Ok(body)) => body, + }; + if body.len() > max_response_bytes { + return Err(Reason::ResponseTooLarge.into()); + } + Ok(body) + } +} + +#[cfg(test)] +mod flow_tests; + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::Arc; + + use futures::executor::block_on; + use http::HeaderMap; + use trusted_server_core::media_signing::SourceVersion; + use trusted_server_core::platform::PlatformHttpClient; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + + use crate::EncypherConfig; + + const SHA: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + fn signer() -> Signer { + let config: EncypherConfig = serde_json::from_value(json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": "cwk_test_worker_token_value", + "max_large_media_bytes": 1_073_741_824_u64, + })) + .expect("should parse config"); + Signer::new(config, 5_000_000) + } + + /// The gate refusal for one job, made before any request is sent. + fn refusal( + reader_url: &str, + source_reference: Option<&str>, + reader_reference: Option<&str>, + origin_host: &str, + ) -> &'static str { + let http = Arc::new(StubHttpClient::new()); + let services = + build_services_with_http_client(Arc::clone(&http) as Arc); + let origin = url::Url::parse("https://origin.example.com/video/ferry.mp4") + .expect("should parse the origin URL"); + let headers = HeaderMap::new(); + let source = VersionedMediaSource::new( + &services, + &origin, + &headers, + "origin", + SourceVersion::new("\"v1\"", 64 * 1024 * 1024).expect("should pin a strong ETag"), + ); + let job = LargeMediaJob { + reader_url, + content_type: "video/mp4", + source: &source, + source_sha256: SHA, + source_reference, + reader_reference, + source_origin_host: origin_host, + remaining: Duration::from_secs(60), + }; + let code = block_on(signer().sign_large(&services, &job)) + .expect_err("should refuse at the gates") + .code(); + assert!( + http.recorded_request_methods().is_empty(), + "should refuse before any request" + ); + code + } + + #[test] + fn a_large_source_without_a_usable_reference_is_refused_before_any_request() { + let reader = "https://publisher.example.com/video/ferry.mp4"; + assert_eq!( + refusal( + "https://cdn.example.org/video/ferry.mp4", + None, + None, + "origin.example.com" + ), + "outside_claim" + ); + assert_eq!( + refusal(reader, None, None, "Origin.example.com:8080"), + "invalid_origin_host" + ); + // A fetched address on another host than the one fetched, and a + // reader address that is not the delivered one, name no source. + assert_eq!( + refusal( + reader, + Some("https://elsewhere.example.com/video/ferry.mp4"), + Some("https://publisher.example.com/video/other.mp4"), + "origin.example.com", + ), + "source_reference_required" + ); + assert_eq!( + refusal(reader, None, None, "origin.example.com"), + "source_reference_required" + ); + } + + #[test] + fn only_a_single_whole_source_range_is_an_unchanged_plan() { + assert!(covers_whole_source(&[json!({"range": [0, 100]})], 100)); + for items in [ + vec![json!({"range": [0, 99]})], + vec![json!({"range": [0, 50]}), json!({"range": [50, 50]})], + vec![json!({"range": [0, 100]}), json!({"literal": "QQ=="})], + vec![json!({"range": [0, 100], "skip": true})], + vec![], + ] { + assert!( + !covers_whole_source(&items, 100), + "should refuse {items:?} as an unchanged plan" + ); + } + } + + #[test] + fn a_source_with_its_own_manifest_is_reported_as_kept_not_failed_or_signed() { + let kept = answer_refusal( + StatusCode::UNPROCESSABLE_ENTITY, + Some("SOURCE_MANIFEST_PRESENT"), + ); + assert_eq!(kept.code(), "source_has_embedded_provenance"); + assert!( + kept.retains_provenance(), + "should report the provenance kept" + ); + for failed in [ + answer_refusal( + StatusCode::UNPROCESSABLE_ENTITY, + Some("HASH_MODE_UNSUPPORTED"), + ), + answer_refusal( + StatusCode::UNPROCESSABLE_ENTITY, + Some("SOURCE_PROVENANCE_INVALID"), + ), + LargeRefusal::Module(Reason::Timeout), + ] { + assert!( + !failed.retains_provenance(), + "{}: should not claim kept provenance", + failed.code() + ); + } + } + + #[test] + fn prepared_errors_map_to_specific_refusals() { + assert_eq!( + answer_refusal(StatusCode::CONFLICT, Some("TRUSTED_SERVER_POLICY_CHANGED")), + LargeRefusal::Module(Reason::PolicyChanged) + ); + assert_eq!( + answer_refusal(StatusCode::CONFLICT, Some("PROBE_VERSION_UNSUPPORTED")).code(), + "probe_release_stale" + ); + assert_eq!( + answer_refusal( + StatusCode::UNPROCESSABLE_ENTITY, + Some("SOURCE_PROVENANCE_INVALID") + ) + .code(), + "source_provenance_invalid" + ); + assert_eq!( + answer_refusal(StatusCode::FORBIDDEN, Some("claim_changed")), + LargeRefusal::Module(Reason::Forbidden) + ); + } + + #[test] + fn capabilities_parse_only_an_enabled_v2_lane() { + let body = |enabled: bool, protocol: &str| { + serde_json::to_vec(&json!({ + "protocol": protocol, + "enabled": enabled, + "policy_digest": "a".repeat(64), + "formats": ["video/mp4"], + "maximums": { + "total_size_bytes": 1_000_000_000_u64, + "structure_decoded_bytes": 33_554_432, + "structure_chunks": 8192, + "manifest_decoded_bytes": 1_048_576, + "prepare_request_bytes": 50_331_648, + }, + "probe": [], + })) + .expect("should encode") + }; + let parsed = Capabilities::parse(&body(true, PROTOCOL)).expect("should parse"); + assert_eq!(parsed.formats, ["video/mp4"]); + assert_eq!(parsed.manifest_decoded_bytes, 1_048_576); + assert!(Capabilities::parse(&body(false, PROTOCOL)).is_none()); + assert!(Capabilities::parse(&body(true, "encypher-prepared-signing/1")).is_none()); + } +} diff --git a/crates/provenance/encypher/src/processor/prepared/flow_tests.rs b/crates/provenance/encypher/src/processor/prepared/flow_tests.rs new file mode 100644 index 0000000000..8ae75e4d41 --- /dev/null +++ b/crates/provenance/encypher/src/processor/prepared/flow_tests.rs @@ -0,0 +1,581 @@ +//! The prepared lane end to end, against a stand-in probe release signed with +//! a test key, a stub service and a stub origin. +//! +//! The stand-in has no format logic: each operation answers JSON the test +//! chose, so these tests exercise the lane's own orchestration and checks, +//! and no Encypher planner code enters this repository. Production always +//! holds releases to the pinned production key; only these tests pass +//! another authority. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use ed25519_dalek::{Signer as _, SigningKey}; +use edgezero_core::key_value_store::KvPage; +use futures::executor::block_on; +use http::HeaderMap; +use serde_json::{Value, json}; +use trusted_server_core::media_signing::{ + LargeMediaJob, LargeSignedMedia, SourceVersion, VersionedMediaSource, +}; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use trusted_server_core::platform::{ + KvError, PlatformHttpClient, PlatformKvStore, RuntimeServices, +}; + +use super::{LargeRefusal, Signer}; +use crate::EncypherConfig; +use crate::media_api::sha256_hex; +use crate::prepared_probe::{ProbeDescriptor, ReleaseAuthority}; + +const POLICY: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const OTHER_POLICY: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const SOURCE_SHA: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; +const SIZE: u64 = 20_000_000; +const READER_URL: &str = "https://publisher.example.com/video/ferry.mp4"; +const ORIGIN_HOST: &str = "publisher.example.com"; +const PLAN_ID: &str = "0f8fad5b-d9cb-469f-a165-70867728950e"; +const STORE: &[u8] = b"jumb-c2pa-remote-manifest-store"; + +const STAND_IN: &str = r#" +(module + (import "env" "host_read" (func $host_read (param i64 i32 i32) (result i32))) + (memory (export "memory") 2) + (global $heap (mut i32) (i32.const 16384)) + (global $result_ptr (mut i32) (i32.const 0)) + (global $result_len (mut i32) (i32.const 0)) + (data (i32.const 1024) "@VERSION@") + (data (i32.const 2048) "@EXTRACT@") + (data (i32.const 4096) "@VERIFY@") + (data (i32.const 6144) "@CHECK@") + (data (i32.const 8192) "@UNKNOWN@") + (func (export "probe_abi") (result i32) (i32.const 1)) + (func (export "probe_alloc") (param $length i32) (result i32) + (local $pointer i32) + (local.set $pointer (global.get $heap)) + (global.set $heap (i32.add (global.get $heap) (local.get $length))) + (local.get $pointer)) + (func (export "probe_free") (param i32 i32)) + (func (export "probe_result_ptr") (result i32) (global.get $result_ptr)) + (func (export "probe_result_len") (result i32) (global.get $result_len)) + (func $answer (param $pointer i32) (param $length i32) (param $code i32) (result i32) + (global.set $result_ptr (local.get $pointer)) + (global.set $result_len (local.get $length)) + (local.get $code)) + (func (export "probe_call") + (param $op i32) (param $op_len i32) (param $arg i32) (param $arg_len i32) (result i32) + (local $kind i32) + (local.set $kind (i32.load8_u (local.get $op))) + ;; "version" (7 bytes) and "verify_plan" (11 bytes) both start with v + (if (i32.and (i32.eq (local.get $kind) (i32.const 118)) (i32.eq (local.get $op_len) (i32.const 7))) + (then (return (call $answer (i32.const 1024) (i32.const @VERSION_LEN@) (i32.const 0))))) + (if (i32.eq (local.get $kind) (i32.const 118)) + (then (return (call $answer (i32.const 4096) (i32.const @VERIFY_LEN@) (i32.const 0))))) + ;; "extract" + (if (i32.eq (local.get $kind) (i32.const 101)) + (then (return (call $answer (i32.const 2048) (i32.const @EXTRACT_LEN@) (i32.const 0))))) + ;; "check_store" + (if (i32.eq (local.get $kind) (i32.const 99)) + (then (return (call $answer (i32.const 6144) (i32.const @CHECK_LEN@) (i32.const 0))))) + (call $answer (i32.const 8192) (i32.const @UNKNOWN_LEN@) (i32.const 1))) +) +"#; + +/// What each stand-in operation answers. +struct Answers { + extract: Value, + verify: Value, + check: Value, +} + +impl Default for Answers { + fn default() -> Self { + Self { + extract: json!({ + "chunks": [{"offset": 0, "data": "AAAA"}], + "has_provenance_reference": false, + }), + verify: json!({"hash_items": [{"range": [0, SIZE]}]}), + check: json!({"ok": true}), + } + } +} + +fn planner() -> String { + "d".repeat(64) +} + +fn wat_string(text: &str) -> String { + text.replace('\\', "\\\\").replace('"', "\\\"") +} + +/// The stand-in release answering `answers`. +fn release(answers: &Answers) -> Vec { + let version = json!({"planner_id": planner()}).to_string(); + let unknown = + json!({"error": {"code": "UNKNOWN_OP", "integrity": false, "message": "no such op"}}) + .to_string(); + let extract = answers.extract.to_string(); + let verify = answers.verify.to_string(); + let check = answers.check.to_string(); + let text = STAND_IN + .replace("@VERSION@", &wat_string(&version)) + .replace("@VERSION_LEN@", &version.len().to_string()) + .replace("@EXTRACT@", &wat_string(&extract)) + .replace("@EXTRACT_LEN@", &extract.len().to_string()) + .replace("@VERIFY@", &wat_string(&verify)) + .replace("@VERIFY_LEN@", &verify.len().to_string()) + .replace("@CHECK@", &wat_string(&check)) + .replace("@CHECK_LEN@", &check.len().to_string()) + .replace("@UNKNOWN@", &wat_string(&unknown)) + .replace("@UNKNOWN_LEN@", &unknown.len().to_string()); + wat::parse_str(&text).expect("should assemble the stand-in release") +} + +fn release_key() -> SigningKey { + SigningKey::from_bytes(&[11; 32]) +} + +/// The descriptor of `artifact`, signed with the test release key. +fn descriptor(artifact: &[u8]) -> ProbeDescriptor { + let mut descriptor = ProbeDescriptor { + abi: 1, + planner_id: planner(), + release: 3, + sha256: sha256_hex(artifact), + signature: String::new(), + }; + let signature = release_key().sign(descriptor.canonical_json().as_bytes()); + descriptor.signature = STANDARD.encode(signature.to_bytes()); + descriptor +} + +fn test_authority() -> ReleaseAuthority { + ReleaseAuthority::pinned(vec![release_key().verifying_key()], 1, Vec::new()) +} + +/// A runtime KV store held in memory. +#[derive(Default)] +struct MemoryKv(Mutex>); + +impl MemoryKv { + fn has(&self, key: &str) -> bool { + self.0.lock().expect("should lock").contains_key(key) + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + Ok(self.0.lock().expect("should lock").get(key).cloned()) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.0 + .lock() + .expect("should lock") + .insert(key.to_owned(), value); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + _ttl: std::time::Duration, + ) -> Result<(), KvError> { + self.put_bytes(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.0.lock().expect("should lock").remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(KvPage::default()) + } +} + +fn signer() -> Signer { + let config: EncypherConfig = serde_json::from_value(json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": "cwk_test_worker_token_value", + "max_large_media_bytes": 1_073_741_824_u64, + })) + .expect("should parse config"); + Signer::new(config, 5_000_000) +} + +fn finalize_answer() -> Value { + json!({ + "protocol": "encypher-prepared-signing/2", + "plan_id": PLAN_ID, + "instance_id": "xmp:iid:0001", + "document_id": "doc-1", + "manifest_disposition": "external", + "content_hash_kind": "hard_binding", + "binding_algorithm": "c2pa.hash.data", + "manifest_store_b64": STANDARD.encode(STORE), + "manifest_sha256": sha256_hex(STORE), + "manifest_url": "https://api.example.com/manifests/doc-1", + "xmp_reference": "declined", + "expected_final_size": SIZE, + "verification_url": "https://verify.example.com/doc-1", + "reused": false, + "policy_digest": POLICY, + }) +} + +fn prepare_answer() -> Value { + json!({ + "protocol": "encypher-prepared-signing/2", + "plan_id": PLAN_ID, + "instance_id": "xmp:iid:0001", + "document_id": "doc-1", + "mime": "video/mp4", + "effective_action": "c2pa.opened", + "geometry": {"disposition": "external"}, + "local_ops": [], + "policy_digest": POLICY, + }) +} + +/// One run of the lane, and what it asked of the service and origin. +struct Run { + result: Result, + http: Arc, + kv: Arc, + signer: Signer, +} + +impl Run { + fn methods(&self) -> Vec { + self.http.recorded_request_methods() + } + + fn refusal(&self) -> LargeRefusal { + *self.result.as_ref().expect_err("should refuse") + } + + fn body(&self, index: usize) -> Value { + serde_json::from_slice(&self.http.recorded_request_bodies()[index]) + .expect("should send JSON") + } +} + +/// The scenario's answers, queued in the order the lane asks for them. +struct Scenario { + answers: Answers, + capabilities_policy: &'static str, + /// The largest manifest store the service says it returns. + manifest_decoded_bytes: usize, + prepare: Value, + finalize: Value, +} + +impl Default for Scenario { + fn default() -> Self { + Self { + answers: Answers::default(), + capabilities_policy: POLICY, + manifest_decoded_bytes: 1_048_576, + prepare: prepare_answer(), + finalize: finalize_answer(), + } + } +} + +fn run(scenario: &Scenario) -> Run { + let artifact = release(&scenario.answers); + let probe = descriptor(&artifact); + let http = Arc::new(StubHttpClient::new()); + http.set_streaming_responses_supported(true); + http.push_response( + 200, + serde_json::to_vec(&json!({"policy_digest": POLICY})).expect("JSON"), + ); + http.push_response( + 200, + serde_json::to_vec(&json!({ + "protocol": "encypher-prepared-signing/2", + "enabled": true, + "policy_digest": scenario.capabilities_policy, + "formats": ["video/mp4"], + "maximums": { + "total_size_bytes": 9_007_199_246_352_383_u64, + "structure_decoded_bytes": 33_554_432, + "structure_chunks": 8192, + "manifest_decoded_bytes": scenario.manifest_decoded_bytes, + "prepare_request_bytes": 50_331_648, + }, + "probe": [probe.to_json()], + })) + .expect("JSON"), + ); + http.push_response(200, artifact); + http.push_response(200, serde_json::to_vec(&scenario.prepare).expect("JSON")); + // The pinned one-byte read that confirms the version before finalize. + http.push_response_with_headers( + 206, + b"\x00".to_vec(), + vec![ + ("content-range", format!("bytes 0-0/{SIZE}")), + ("content-length", "1".to_owned()), + ("etag", "\"v1\"".to_owned()), + ], + ); + http.push_response(200, serde_json::to_vec(&scenario.finalize).expect("JSON")); + + let kv = Arc::new(MemoryKv::default()); + let services: RuntimeServices = + build_services_with_http_client(Arc::clone(&http) as Arc) + .with_kv_store(Arc::clone(&kv) as Arc); + let origin = url::Url::parse("https://publisher.example.com/video/ferry.mp4") + .expect("should parse the origin URL"); + let headers = HeaderMap::new(); + let source = VersionedMediaSource::new( + &services, + &origin, + &headers, + "origin", + SourceVersion::new("\"v1\"", SIZE).expect("should pin a strong ETag"), + ); + let job = LargeMediaJob { + reader_url: READER_URL, + content_type: "video/mp4", + source: &source, + source_sha256: SOURCE_SHA, + source_reference: Some(READER_URL), + reader_reference: None, + source_origin_host: ORIGIN_HOST, + remaining: std::time::Duration::from_secs(60), + }; + let signer = signer(); + let result = block_on(signer.sign_large_with(&services, &job, &test_authority())); + Run { + result, + http, + kv, + signer, + } +} + +#[test] +fn an_unchanged_source_is_bound_whole_after_its_version_is_confirmed() { + let run = run(&Scenario::default()); + + run.result.as_ref().expect("should sign"); + assert_eq!( + run.methods(), + ["GET", "GET", "GET", "POST", "GET", "POST"], + "should confirm the pinned version after prepare and before finalize" + ); + let finalize = run.body(5); + assert_eq!( + finalize["binding"], + json!({"type": "data_hash", "digest": SOURCE_SHA}), + "should bind the whole-source digest" + ); + assert_eq!( + finalize["xmp_reference"], false, + "should never edit the source" + ); + let keys: Vec = run + .http + .recorded_request_headers() + .iter() + .filter_map(|headers| { + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("idempotency-key")) + .map(|(_, value)| value.clone()) + }) + .collect(); + assert_eq!(keys.len(), 2, "should key prepare and finalize"); + assert_eq!(keys[0], keys[1], "should finalize under the plan's own key"); +} + +#[test] +fn a_manifest_larger_than_the_lane_takes_is_refused() { + let run = run(&Scenario { + manifest_decoded_bytes: STORE.len() - 1, + ..Scenario::default() + }); + assert_eq!( + run.refusal().code(), + "manifest_too_large", + "should hold the store to the smaller of the service's and the edge's limit" + ); +} + +#[test] +fn a_source_with_provenance_of_its_own_keeps_it_and_is_never_prepared() { + for (extract, reason) in [ + ( + json!({ + "chunks": [{"offset": 0, "data": "AAAA"}], + "credentials": {"manifests": 1}, + "has_provenance_reference": false, + }), + "source_has_embedded_provenance", + ), + ( + json!({ + "chunks": [{"offset": 0, "data": "AAAA"}], + "has_provenance_reference": true, + }), + "source_has_provenance_reference", + ), + ] { + let run = run(&Scenario { + answers: Answers { + extract, + ..Answers::default() + }, + ..Scenario::default() + }); + + let refusal = run.refusal(); + assert_eq!(refusal.code(), reason); + assert!(refusal.retains_provenance(), "{reason}: should keep it"); + assert!( + !run.methods().iter().any(|method| method == "POST"), + "{reason}: should never prepare or finalize" + ); + } +} + +#[test] +fn a_release_that_does_not_report_a_provenance_reference_is_not_trusted() { + for flag in [None, Some(json!("false")), Some(json!(0))] { + let mut extract = json!({"chunks": [{"offset": 0, "data": "AAAA"}]}); + if let Some(flag) = flag.clone() { + extract["has_provenance_reference"] = flag; + } + let run = run(&Scenario { + answers: Answers { + extract, + ..Answers::default() + }, + ..Scenario::default() + }); + + let refusal = run.refusal(); + assert_eq!(refusal.code(), "probe_result_invalid", "{flag:?}"); + assert!(!refusal.retains_provenance(), "{flag:?}: should not claim"); + assert!( + !run.methods().iter().any(|method| method == "POST"), + "{flag:?}: should never prepare" + ); + } +} + +#[test] +fn a_plan_that_is_not_the_unchanged_whole_source_is_never_finalized() { + let mut edits = prepare_answer(); + edits["local_ops"] = json!([{"op": "xmp"}]); + let mut edited = prepare_answer(); + edited["effective_action"] = json!("c2pa.edited"); + let mut embedded = prepare_answer(); + embedded["geometry"] = json!({"disposition": "embedded"}); + for prepare in [edits, edited, embedded] { + let run = run(&Scenario { + prepare, + ..Scenario::default() + }); + assert_eq!(run.refusal().code(), "prepare_answer_invalid"); + assert_eq!(run.methods().len(), 4, "should stop after prepare"); + } + + let run = run(&Scenario { + answers: Answers { + verify: json!({"hash_items": [{"range": [0, SIZE - 1]}]}), + ..Answers::default() + }, + ..Scenario::default() + }); + assert_eq!(run.refusal().code(), "plan_not_whole_source"); + assert_eq!( + run.methods().len(), + 4, + "should stop before the version check" + ); +} + +#[test] +fn a_finalized_answer_that_does_not_bind_the_source_is_never_delivered() { + let mut resized = finalize_answer(); + resized["expected_final_size"] = json!(SIZE + 1); + let mut swapped = finalize_answer(); + swapped["manifest_sha256"] = json!(sha256_hex(b"another store")); + let mut embedded = finalize_answer(); + embedded["manifest_disposition"] = json!("embedded"); + for (finalize, reason) in [ + (resized, "finalize_answer_invalid"), + (swapped, "signed_digest_mismatch"), + (embedded, "finalize_answer_invalid"), + ] { + let run = run(&Scenario { + finalize, + ..Scenario::default() + }); + assert_eq!(run.refusal().code(), reason); + } + + let run = run(&Scenario { + answers: Answers { + check: json!({"ok": false}), + ..Answers::default() + }, + ..Scenario::default() + }); + assert_eq!( + run.refusal().code(), + "signed_digest_mismatch", + "should refuse a store that does not bind the whole-source digest" + ); +} + +#[test] +fn a_changed_policy_evicts_the_cached_policy_and_publishes_nothing() { + let changed = run(&Scenario { + capabilities_policy: OTHER_POLICY, + ..Scenario::default() + }); + assert_eq!(changed.refusal().code(), "policy_changed"); + assert!( + !changed.kv.has(&changed.signer.policy_key), + "should evict the policy the lane held" + ); + assert!(!changed.methods().iter().any(|method| method == "POST")); + + let mut prepared = prepare_answer(); + prepared["policy_digest"] = json!(OTHER_POLICY); + let mismatched = run(&Scenario { + prepare: prepared, + ..Scenario::default() + }); + assert_eq!(mismatched.refusal().code(), "policy_digest_mismatch"); + assert!(!mismatched.kv.has(&mismatched.signer.policy_key)); + + let mut finalized = finalize_answer(); + finalized["policy_digest"] = json!(OTHER_POLICY); + let late = run(&Scenario { + finalize: finalized, + ..Scenario::default() + }); + assert_eq!(late.refusal().code(), "policy_digest_mismatch"); + assert!(!late.kv.has(&late.signer.policy_key)); +} diff --git a/crates/provenance/encypher/src/svg_equivalence.rs b/crates/provenance/encypher/src/svg_equivalence.rs new file mode 100644 index 0000000000..3f4a9d0685 --- /dev/null +++ b/crates/provenance/encypher/src/svg_equivalence.rs @@ -0,0 +1,977 @@ +//! The standalone SVG document an inline `` is signed as. +//! +//! A browser builds an inline SVG's tree with the HTML parser, which reads +//! it differently from an XML parser: it needs no namespace declaration, +//! keeps the first of two attributes whose names differ only in case, and +//! decodes HTML character references. What is signed must be a standalone +//! SVG document whose XML tree is the tree the browser built, so a verifier +//! reading the signed bytes sees what the reader saw. +//! +//! The fragment is parsed as HTML in a `body` context, into scraper's +//! namespace-aware tree, as the browser parsed it, and refused unless that +//! yields exactly one `` root with nothing displaced beside it but +//! whitespace. Where the original bytes already parse as XML to the same +//! tree, they are signed unchanged. Otherwise, unless the fragment carries +//! provenance of its own, which is never rewritten, the browser's tree is +//! serialized as XML and kept only when that serialization parses to the +//! same tree both as HTML and as XML. +//! +//! Trees are compared by element namespace and local name, the set of +//! ordinary attributes by namespace, local name and value, text with +//! adjacent text merged and CDATA read as text, and comments. Namespace +//! declarations are not ordinary attributes and are not compared: only the +//! namespaces they resolve are. Anything else, a processing instruction or a +//! doctype, makes two trees unequal. XML is always parsed with DTDs +//! refused, so no entity is ever declared or resolved. + +use std::borrow::Cow; +use std::marker::PhantomData; + +use ego_tree::NodeRef; +use ego_tree::iter::Children; +use html5ever::tendril::TendrilSink as _; +use html5ever::{LocalName, Namespace, ParseOpts, QualName, parse_fragment}; +use scraper::node::Element; +use scraper::{HtmlTreeSink, Node}; + +use crate::embedded; +use crate::processor::Reason; + +const HTML_NAMESPACE: &str = "http://www.w3.org/1999/xhtml"; +const SVG_NAMESPACE: &str = "http://www.w3.org/2000/svg"; +const XLINK_NAMESPACE: &str = "http://www.w3.org/1999/xlink"; +const XML_NAMESPACE: &str = "http://www.w3.org/XML/1998/namespace"; +const XMLNS_NAMESPACE: &str = "http://www.w3.org/2000/xmlns/"; + +/// The most nodes an inline SVG may hold. +const MAX_NODES: usize = 10_000; +/// The deepest an inline SVG's elements may nest. +const MAX_DEPTH: usize = 256; +/// Nodes the HTML fragment parser makes besides the fragment's own: the +/// fragment root, the context element and the root `html` element. +const PARSER_NODES: usize = 3; + +/// The declarations the serialization's root carries. +const SVG_DECLARATION: &str = " xmlns=\"http://www.w3.org/2000/svg\""; +const XLINK_DECLARATION: &str = " xmlns:xlink=\"http://www.w3.org/1999/xlink\""; + +/// The standalone SVG document `fragment`, an inline `` element as +/// written in an HTML page, is signed as: `fragment` itself where it +/// already parses as XML to the tree the browser built from it, else that +/// tree serialized as XML. +/// +/// # Errors +/// +/// - [`Reason::MediaTooLarge`] where `fragment` is longer than `max_bytes`. +/// - [`Reason::SvgProvenanceNotEquivalent`] where `fragment` carries +/// provenance of its own and is not, as written, equivalent: it is never +/// rewritten. +/// - [`Reason::InlineSvgNotEquivalent`] where the browser's tree is not one +/// `` root, exceeds the node or depth bound, or has no XML +/// serialization that parses back to it as both HTML and XML. +pub(crate) fn signable_svg(fragment: &str, max_bytes: usize) -> Result, Reason> { + if fragment.len() > max_bytes { + return Err(Reason::MediaTooLarge); + } + let parsed = parse_html(fragment).ok_or(Reason::InlineSvgNotEquivalent)?; + let root = parsed.root().ok_or(Reason::InlineSvgNotEquivalent)?; + if let Some(document) = parse_xml(fragment) + && equal(&Browser, root, &Xml(PhantomData), document.root_element()) + { + return Ok(Cow::Borrowed(fragment)); + } + if carries_provenance(fragment, root) { + return Err(Reason::SvgProvenanceNotEquivalent); + } + let serialized = serialize(root, max_bytes).ok_or(Reason::InlineSvgNotEquivalent)?; + let equivalent = { + let as_html = parse_html(&serialized); + let as_xml = parse_xml(&serialized); + match (as_html.as_ref().and_then(Parsed::root), as_xml) { + (Some(as_html), Some(as_xml)) => { + equal(&Browser, root, &Browser, as_html) + && equal(&Browser, root, &Xml(PhantomData), as_xml.root_element()) + } + _ => false, + } + }; + if equivalent { + Ok(Cow::Owned(serialized)) + } else { + Err(Reason::InlineSvgNotEquivalent) + } +} + +/// Refuse all edits inside an original inline SVG that carries provenance. +/// A fragment whose structure cannot be inspected within the parser's limits +/// is also left intact, rather than assuming it has no provenance. +pub(crate) fn descendant_edit_refusal(fragment: &str) -> Option { + let Some(parsed) = parse_html(fragment) else { + return Some(Reason::InlineSvgNotEquivalent); + }; + let Some(root) = parsed.root() else { + return Some(Reason::InlineSvgNotEquivalent); + }; + carries_provenance(fragment, root).then_some(Reason::SvgExistingProvenance) +} + +/// Whether the inline SVG under `root` carries provenance of its own: the +/// manifest carrier the signer's writer finds, an element or attribute in +/// the `c2pa` prefix, a declaration of a C2PA namespace, or a provenance +/// reference such as XMP's `dcterms:provenance`. The HTML parser keeps such +/// prefixed names whole, as written. Ordinary text that mentions C2PA is +/// not provenance. +fn carries_provenance(fragment: &str, root: NodeRef<'_, Node>) -> bool { + if embedded::existing_carrier(fragment.as_bytes()).is_some() { + return true; + } + root.descendants().any(|node| { + let Node::Element(element) = node.value() else { + return false; + }; + names_provenance(&element.name.local) + || element.attrs.iter().any(|(name, value)| { + names_provenance(&name.local) + || (is_declaration(&name.ns, &name.local) && is_c2pa_namespace(value)) + }) + }) +} + +/// Whether a name the HTML parser kept whole, prefix included, is in the +/// `c2pa` prefix or names a provenance reference. +fn names_provenance(name: &str) -> bool { + let (prefix, local) = name.rsplit_once(':').unwrap_or(("", name)); + prefix.eq_ignore_ascii_case("c2pa") || local.eq_ignore_ascii_case("provenance") +} + +fn is_c2pa_namespace(uri: &str) -> bool { + let uri = uri.trim(); + ["http://c2pa.org/", "https://c2pa.org/"] + .iter() + .any(|start| { + uri.get(..start.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(start)) + }) +} + +/// scraper's tree for one fragment, with its `` root. +struct Parsed { + html: scraper::Html, + root: ego_tree::NodeId, +} + +impl Parsed { + fn root(&self) -> Option> { + self.html.tree.get(self.root) + } +} + +/// `fragment` parsed as HTML in a `body` context with scripting on, as a +/// browser parses inline SVG, into scraper's tree, or `None` unless that +/// yields exactly one `` element at the top, with only whitespace text +/// beside it, within the node bound. +fn parse_html(fragment: &str) -> Option { + // Every element and comment the tokenizer starts begins at a `<`, so + // this bounds what the parse allocates before the exact count below. + if fragment.bytes().filter(|byte| *byte == b'<').count() > MAX_NODES { + return None; + } + let context = QualName::new( + None, + Namespace::from(HTML_NAMESPACE), + LocalName::from("body"), + ); + let html = parse_fragment( + HtmlTreeSink::new(scraper::Html::new_fragment()), + ParseOpts::default(), + context, + Vec::new(), + true, + ) + .one(fragment); + if html.tree.values().count() > MAX_NODES + PARSER_NODES { + return None; + } + let top = html + .tree + .root() + .children() + .find(|child| child.value().is_element())?; + let mut root = None; + for child in top.children() { + match child.value() { + Node::Element(element) + if root.is_none() + && &*element.name.ns == SVG_NAMESPACE + && &*element.name.local == "svg" => + { + root = Some(child.id()); + } + Node::Text(text) if text.chars().all(is_html_whitespace) => {} + _ => return None, + } + } + let root = root?; + Some(Parsed { html, root }) +} + +fn is_html_whitespace(character: char) -> bool { + matches!(character, '\t' | '\n' | '\u{c}' | '\r' | ' ') +} + +/// `text` parsed as XML with DTDs refused, so no entity is declared or +/// resolved, or `None` where it is not well-formed. +fn parse_xml(text: &str) -> Option> { + let options = roxmltree::ParsingOptions { + allow_dtd: false, + nodes_limit: u32::try_from(MAX_NODES + 1).unwrap_or(u32::MAX), + ..roxmltree::ParsingOptions::default() + }; + roxmltree::Document::parse_with_options(text, options).ok() +} + +/// What a compared node is. +enum Kind<'a> { + Element { namespace: &'a str, name: &'a str }, + Text(&'a str), + Comment(&'a str), + Other, +} + +/// A tree, as the comparison walks it. +trait View<'a> { + type Node: Copy; + type Children: Iterator; + + fn kind(&self, node: Self::Node) -> Kind<'a>; + /// The element's ordinary attributes as (namespace, local name, value), + /// sorted, namespace declarations excluded. + fn attributes(&self, node: Self::Node) -> Vec<(&'a str, &'a str, &'a str)>; + fn children(&self, node: Self::Node) -> Self::Children; +} + +/// Whether an attribute is a namespace declaration rather than an ordinary +/// attribute. The HTML parser puts `xmlns` and `xmlns:xlink` in the XMLNS +/// namespace and leaves any other `xmlns:` prefixed name unnamespaced. +fn is_declaration(namespace: &str, name: &str) -> bool { + namespace == XMLNS_NAMESPACE + || (namespace.is_empty() && (name == "xmlns" || name.starts_with("xmlns:"))) +} + +/// scraper's tree, whose node references carry the tree. +struct Browser; + +impl<'a> View<'a> for Browser { + type Node = NodeRef<'a, Node>; + type Children = Children<'a, Node>; + + fn kind(&self, node: Self::Node) -> Kind<'a> { + match node.value() { + Node::Element(element) => Kind::Element { + namespace: &element.name.ns, + name: &element.name.local, + }, + Node::Text(text) => Kind::Text(&text.text), + Node::Comment(comment) => Kind::Comment(&comment.comment), + _ => Kind::Other, + } + } + + fn attributes(&self, node: Self::Node) -> Vec<(&'a str, &'a str, &'a str)> { + let Node::Element(element) = node.value() else { + return Vec::new(); + }; + let mut ordinary: Vec<(&'a str, &'a str, &'a str)> = element + .attrs + .iter() + .map(|(name, value)| (&*name.ns, &*name.local, &**value)) + .filter(|(namespace, name, _)| !is_declaration(namespace, name)) + .collect(); + ordinary.sort_unstable(); + ordinary + } + + fn children(&self, node: Self::Node) -> Self::Children { + node.children() + } +} + +/// A roxmltree document, whose nodes carry their own references. +struct Xml<'a, 'input: 'a>(PhantomData>); + +impl<'a, 'input: 'a> View<'a> for Xml<'a, 'input> { + type Node = roxmltree::Node<'a, 'input>; + type Children = roxmltree::Children<'a, 'input>; + + fn kind(&self, node: Self::Node) -> Kind<'a> { + match node.node_type() { + roxmltree::NodeType::Element => { + let name = node.tag_name(); + Kind::Element { + namespace: name.namespace().unwrap_or_default(), + name: name.name(), + } + } + roxmltree::NodeType::Text => Kind::Text(node.text().unwrap_or_default()), + roxmltree::NodeType::Comment => Kind::Comment(node.text().unwrap_or_default()), + roxmltree::NodeType::Root | roxmltree::NodeType::PI => Kind::Other, + } + } + + fn attributes(&self, node: Self::Node) -> Vec<(&'a str, &'a str, &'a str)> { + // roxmltree keeps namespace declarations apart from attributes. + let mut ordinary: Vec<_> = node + .attributes() + .map(|attribute| { + ( + attribute.namespace().unwrap_or_default(), + attribute.name(), + attribute.value(), + ) + }) + .collect(); + ordinary.sort_unstable(); + ordinary + } + + fn children(&self, node: Self::Node) -> Self::Children { + node.children() + } +} + +/// Whether the trees under `a_root` and `b_root` are equal, within the +/// depth bound. The walk keeps one pair of child iterators per open element +/// and copies neither tree. +fn equal<'a, 'b, A: View<'a>, B: View<'b>>(a: &A, a_root: A::Node, b: &B, b_root: B::Node) -> bool { + if !same(a, a_root, b, b_root) { + return false; + } + let mut open = vec![(a.children(a_root), b.children(b_root))]; + while let Some((mut a_children, mut b_children)) = open.pop() { + match (a_children.next(), b_children.next()) { + (None, None) => {} + (Some(a_node), Some(b_node)) => { + if !same(a, a_node, b, b_node) { + return false; + } + let element = matches!(a.kind(a_node), Kind::Element { .. }); + open.push((a_children, b_children)); + if element { + if open.len() >= MAX_DEPTH { + return false; + } + open.push((a.children(a_node), b.children(b_node))); + } + } + _ => return false, + } + } + true +} + +fn same<'a, 'b, A: View<'a>, B: View<'b>>(a: &A, a_node: A::Node, b: &B, b_node: B::Node) -> bool { + match (a.kind(a_node), b.kind(b_node)) { + ( + Kind::Element { + namespace: a_namespace, + name: a_name, + }, + Kind::Element { + namespace: b_namespace, + name: b_name, + }, + ) => { + a_namespace == b_namespace + && a_name == b_name + && a.attributes(a_node) == b.attributes(b_node) + } + (Kind::Text(a_text), Kind::Text(b_text)) + | (Kind::Comment(a_text), Kind::Comment(b_text)) => a_text == b_text, + _ => false, + } +} + +/// The tree under `root` written as a standalone SVG document, or `None` +/// where XML cannot carry it: an element outside the SVG namespace, an +/// attribute outside no namespace, `XLink` or XML, a name XML cannot spell, a +/// character XML forbids, a comment XML cannot hold, any other node, or +/// more than `max_bytes`. +/// +/// The root declares the SVG namespace, and the `XLink` prefix where an +/// attribute uses it. Every element and ordinary attribute is the tree's, +/// with its value, which is the HTML parser's: the first of attributes that +/// differ only in case, and character references decoded. Values escape +/// `&`, `<`, `>`, `"`, tab, line feed and carriage return, and text escapes +/// `&`, `<`, `>` and carriage return, so XML's attribute-value and line-end +/// normalization leaves them as parsed. +fn serialize(root: NodeRef<'_, Node>, max_bytes: usize) -> Option { + let Node::Element(element) = root.value() else { + return None; + }; + let mut output = SvgOutput::new(max_bytes); + let mut uses_xlink = false; + write_start(&mut output, element, &mut uses_xlink)?; + // The root's declarations follow its name: the XLink one is known only + // once every attribute is written. + let declarations = 1 + element.name.local.len(); + output.insert_str(declarations, SVG_DECLARATION)?; + let mut open: Vec<(Children<'_, Node>, &str)> = Vec::new(); + if root.first_child().is_some() { + output.push('>')?; + open.push((root.children(), &*element.name.local)); + } else { + output.push_str("/>")?; + } + while let Some((mut children, name)) = open.pop() { + let Some(child) = children.next() else { + output.push_str("')?; + continue; + }; + open.push((children, name)); + match child.value() { + Node::Element(element) => { + write_start(&mut output, element, &mut uses_xlink)?; + if child.first_child().is_some() { + if open.len() >= MAX_DEPTH { + return None; + } + output.push('>')?; + open.push((child.children(), &*element.name.local)); + } else { + output.push_str("/>")?; + } + } + Node::Text(text) => write_text(&mut output, &text.text)?, + Node::Comment(comment) => write_comment(&mut output, &comment.comment)?, + _ => return None, + } + } + if uses_xlink { + output.insert_str(declarations + SVG_DECLARATION.len(), XLINK_DECLARATION)?; + } + Some(output.text) +} + +/// A serializer buffer whose length and requested allocation never exceed +/// the output limit. Grow geometrically, but request the exact bounded target +/// so String's implicit doubling cannot allocate beyond it. +struct SvgOutput { + text: String, + max_bytes: usize, +} + +impl SvgOutput { + fn new(max_bytes: usize) -> Self { + Self { + text: String::new(), + max_bytes, + } + } + + fn reserve(&mut self, additional: usize) -> Option<()> { + let needed = self.text.len().checked_add(additional)?; + if needed > self.max_bytes { + return None; + } + if needed > self.text.capacity() { + let target = self + .text + .capacity() + .checked_mul(2) + .unwrap_or(self.max_bytes) + .max(64.min(self.max_bytes)) + .max(needed) + .min(self.max_bytes); + self.text + .try_reserve_exact(target.checked_sub(self.text.len())?) + .ok()?; + } + Some(()) + } + + fn push_str(&mut self, text: &str) -> Option<()> { + self.reserve(text.len())?; + self.text.push_str(text); + Some(()) + } + + fn push(&mut self, character: char) -> Option<()> { + self.reserve(character.len_utf8())?; + self.text.push(character); + Some(()) + } + + fn insert_str(&mut self, at: usize, text: &str) -> Option<()> { + self.reserve(text.len())?; + self.text.insert_str(at, text); + Some(()) + } +} + +/// Writes ` Option<()> { + if &*element.name.ns != SVG_NAMESPACE || !is_xml_local_name(&element.name.local) { + return None; + } + output.push('<')?; + output.push_str(&element.name.local)?; + for (name, value) in element.attrs.iter() { + let namespace = &*name.ns; + let local = &*name.local; + if is_declaration(namespace, local) { + continue; + } + let prefix = match namespace { + "" => "", + XLINK_NAMESPACE => { + *uses_xlink = true; + "xlink:" + } + XML_NAMESPACE => "xml:", + _ => return None, + }; + if !is_xml_local_name(local) { + return None; + } + output.push(' ')?; + output.push_str(prefix)?; + output.push_str(local)?; + output.push_str("=\"")?; + for character in value.chars() { + match character { + '&' => output.push_str("&"), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + '"' => output.push_str("""), + '\t' => output.push_str(" "), + '\n' => output.push_str(" "), + '\r' => output.push_str(" "), + character if is_xml_char(character) => output.push(character), + _ => return None, + }?; + } + output.push('"')?; + } + Some(()) +} + +fn write_text(output: &mut SvgOutput, text: &str) -> Option<()> { + for character in text.chars() { + match character { + '&' => output.push_str("&"), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + '\r' => output.push_str(" "), + character if is_xml_char(character) => output.push(character), + _ => return None, + }?; + } + Some(()) +} + +/// Writes a comment XML can hold: no `--` inside, no `-` at the end. +fn write_comment(output: &mut SvgOutput, text: &str) -> Option<()> { + if text.contains("--") || text.ends_with('-') || !text.chars().all(is_xml_char) { + return None; + } + output.push_str("")?; + Some(()) +} + +/// A character XML 1.0 allows in a document. +fn is_xml_char(character: char) -> bool { + matches!( + character, + '\t' | '\n' + | '\r' + | '\u{20}'..='\u{d7ff}' + | '\u{e000}'..='\u{fffd}' + | '\u{10000}'..='\u{10ffff}' + ) +} + +/// A name XML can write without a prefix. Conservative: a name it refuses +/// is never written, and anything this admits that XML reads otherwise +/// fails the equivalence check. +fn is_xml_local_name(name: &str) -> bool { + let mut characters = name.chars(); + characters + .next() + .is_some_and(|first| first.is_alphabetic() || first == '_') + && characters + .all(|character| character.is_alphanumeric() || matches!(character, '-' | '_' | '.')) +} + +#[cfg(test)] +mod tests { + use super::*; + + const LIMIT: usize = 1024 * 1024; + + fn signed(fragment: &str) -> Result, Reason> { + signable_svg(fragment, LIMIT) + } + + /// The serialization `fragment` is signed as, checked to be a standalone + /// SVG document whose tree, read as XML and again as HTML, is the + /// browser's tree of `fragment`. + fn normalized(fragment: &str) -> String { + let Ok(Cow::Owned(serialized)) = signed(fragment) else { + panic!("should serialize {fragment}, got {:?}", signed(fragment)); + }; + let browser = parse_html(fragment).expect("should parse the original as HTML"); + let browser = browser.root().expect("should have an svg root"); + let reparsed = parse_html(&serialized).expect("should parse the serialization as HTML"); + let reparsed = reparsed.root().expect("should keep one svg root"); + let document = parse_xml(&serialized).expect("should be standalone XML"); + assert_eq!( + document.root_element().tag_name().namespace(), + Some(SVG_NAMESPACE), + "should declare the SVG namespace" + ); + assert!( + equal(&Browser, browser, &Browser, reparsed), + "should read back as the browser's tree: {serialized}" + ); + assert!( + equal( + &Browser, + browser, + &Xml(PhantomData), + document.root_element() + ), + "should read as XML to the browser's tree: {serialized}" + ); + serialized + } + + fn assert_unchanged(fragment: &str) { + assert!( + matches!(signed(fragment), Ok(Cow::Borrowed(kept)) if kept == fragment), + "should sign {fragment} as written, got {:?}", + signed(fragment) + ); + } + + /// The first SVG element named `name` in `serialized`, read as XML. + fn element<'a, 'input>( + document: &'a roxmltree::Document<'input>, + name: &str, + ) -> roxmltree::Node<'a, 'input> { + document + .descendants() + .find(|node| node.has_tag_name((SVG_NAMESPACE, name))) + .unwrap_or_else(|| panic!("should hold a {name} element")) + } + + #[test] + fn the_first_of_two_case_variant_attributes_is_the_one_signed() { + // The captured logo: the browser keeps `viewbox`'s value as + // `viewBox` and drops the later `viewBox`. + let serialized = normalized( + "\n \n", + ); + let document = parse_xml(&serialized).expect("should be XML"); + let root = document.root_element(); + assert_eq!(root.attribute("viewBox"), Some("0 0 300 300.251")); + assert_eq!(root.attribute("viewbox"), None); + assert_eq!(root.attribute("version"), Some("1.1")); + assert_eq!(element(&document, "path").attribute("d"), Some("M0 0h1")); + } + + #[test] + fn an_svg_without_its_namespace_is_signed_in_it() { + let serialized = normalized( + "", + ); + let document = parse_xml(&serialized).expect("should be XML"); + let root = document.root_element(); + assert_eq!(root.attribute("fill"), Some("currentColor")); + assert_eq!(root.attribute("aria-hidden"), Some("true")); + assert_eq!( + element(&document, "path").attribute("d"), + Some("M18.244 2.25h3.308") + ); + } + + #[test] + fn a_standalone_svg_is_signed_as_written_whatever_its_declarations() { + assert_unchanged( + "", + ); + assert_unchanged( + "", + ); + assert_unchanged( + "", + ); + assert_unchanged( + "&’&", + ); + } + + #[test] + fn an_xlink_attribute_is_signed_in_the_xlink_namespace() { + for fragment in [ + "", + // A wrong binding is a declaration the browser ignores. + "", + ] { + let serialized = normalized(fragment); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!( + element(&document, "use").attribute((XLINK_NAMESPACE, "href")), + Some("#a"), + "for {fragment}" + ); + } + assert!( + !normalized("").contains("xmlns:xlink"), + "should declare XLink only where it is used" + ); + } + + #[test] + fn wrapped_attribute_whitespace_keeps_its_parsed_value() { + let serialized = normalized( + "", + ); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!( + element(&document, "path").attribute("d"), + Some("M0 0\n L1 1\tZ") + ); + } + + #[test] + fn html_character_references_are_signed_as_their_characters() { + let serialized = normalized( + "A &<B’", + ); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!(element(&document, "title").text(), Some("A\u{a0}&a�b"); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!(element(&document, "text").text(), Some("a\u{fffd}b")); + } + + #[test] + fn cdata_is_signed_as_its_text() { + let serialized = normalized(""); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!(element(&document, "style").text(), Some("a&e;"); + assert!(!serialized.contains("DOCTYPE"), "should carry no doctype"); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!(element(&document, "text").text(), Some("&e;")); + } + + #[test] + fn markup_that_breaks_out_of_the_svg_is_refused() { + for fragment in [ + "

x

", + "
x
", + "
x
", + "x", + ] { + assert_eq!( + signed(fragment), + Err(Reason::InlineSvgNotEquivalent), + "should refuse {fragment}" + ); + } + } + + #[test] + fn what_xml_cannot_carry_is_refused() { + for fragment in [ + "", + "", + "", + "￾", + "", + "", + ] { + assert_eq!( + signed(fragment), + Err(Reason::InlineSvgNotEquivalent), + "should refuse {fragment}" + ); + } + } + + #[test] + fn provenance_the_svg_carries_is_never_rewritten() { + for fragment in [ + // The writer's carrier, without and with the SVG namespace. + "QUJD", + "QUJD", + // An unclosed carrier element. + "QUJD", + // A C2PA namespace declared for anything. + "", + // An XMP provenance reference, which need not name C2PA. + "", + "https://example.com/m", + ] { + assert_eq!( + signed(fragment), + Err(Reason::SvgProvenanceNotEquivalent), + "should leave {fragment} as written" + ); + } + } + + #[test] + fn a_caption_that_mentions_c2pa_is_not_provenance() { + let serialized = normalized( + "Signed with C2PAc2pa", + ); + let document = parse_xml(&serialized).expect("should be XML"); + assert_eq!(element(&document, "title").text(), Some("Signed with C2PA")); + } + + #[test] + fn serializer_checks_root_attributes_text_and_namespace_growth_before_writing() { + let fragment = ">&"; + let parsed = parse_html(fragment).expect("should parse"); + let root = parsed.root().expect("should have a root"); + let serialized = serialize(root, LIMIT).expect("should serialize"); + assert!(serialized.contains(SVG_DECLARATION)); + assert!(serialized.contains(XLINK_DECLARATION)); + for limit in 0..serialized.len() { + assert!( + serialize(root, limit).is_none(), + "must refuse limit {limit}" + ); + } + assert_eq!(serialize(root, serialized.len()), Some(serialized)); + + let fragment = format!("", "\"".repeat(1000)); + let parsed = parse_html(&fragment).expect("should parse a large root attribute"); + let Node::Element(element) = parsed.root().expect("should have a root").value() else { + panic!("should be an element"); + }; + let mut output = SvgOutput::new(127); + assert!(write_start(&mut output, element, &mut false).is_none()); + assert!(output.text.len() <= 127); + assert!(output.text.capacity() <= 127); + assert_eq!( + signable_svg(&fragment, 1100), + Err(Reason::InlineSvgNotEquivalent) + ); + + let fragment = format!("{}", ">".repeat(1000)); + assert_eq!( + signable_svg(&fragment, 1100), + Err(Reason::InlineSvgNotEquivalent) + ); + } + + #[test] + fn serializer_buffer_growth_is_bounded_amortized_and_not_the_media_limit() { + let mut output = SvgOutput::new(5_000_000); + assert_eq!(output.text.capacity(), 0); + output.push_str("", ">"), + ("\r", " "), + ] { + for limit in 0..escaped.len() { + let mut output = SvgOutput::new(limit); + assert!(write_text(&mut output, text).is_none()); + assert!(output.text.len() <= limit); + assert!(output.text.capacity() <= limit); + } + let mut output = SvgOutput::new(escaped.len()); + write_text(&mut output, text).expect("should fit the complete escape"); + assert_eq!(output.text, escaped); + } + for limit in 0..10 { + let mut output = SvgOutput::new(limit); + assert!(write_comment(&mut output, "art").is_none()); + assert!(output.text.len() <= limit); + assert!(output.text.capacity() <= limit); + } + let mut output = SvgOutput::new(10); + write_comment(&mut output, "art").expect("should fit the complete comment"); + assert_eq!(output.text, ""); + } + + #[test] + fn the_size_node_and_depth_bounds_refuse() { + let fragment = ""; + assert_eq!( + signable_svg(fragment, fragment.len() - 1), + Err(Reason::MediaTooLarge) + ); + let wide = format!( + "{}", + "".repeat(MAX_NODES) + ); + assert_eq!(signed(&wide), Err(Reason::InlineSvgNotEquivalent)); + let deep = format!( + "{}{}", + "".repeat(MAX_DEPTH), + "".repeat(MAX_DEPTH) + ); + assert_eq!(signed(&deep), Err(Reason::InlineSvgNotEquivalent)); + let nested = format!( + "{}{}", + "".repeat(MAX_DEPTH - 2), + "".repeat(MAX_DEPTH - 2) + ); + assert_unchanged(&nested); + } +} diff --git a/crates/provenance/encypher/src/tests.rs b/crates/provenance/encypher/src/tests.rs index 6740aedf93..bd2ddb588d 100644 --- a/crates/provenance/encypher/src/tests.rs +++ b/crates/provenance/encypher/src/tests.rs @@ -2,6 +2,8 @@ use super::*; use std::sync::Arc; +use base64::Engine as _; + use edgezero_core::body::Body as EdgeBody; use edgezero_core::key_value_store::KvPage; use futures::executor::block_on; @@ -87,13 +89,36 @@ fn an_unselected_module_registers_nothing() { } #[test] -fn a_selected_and_placed_module_registers_its_middleware() { +fn a_selected_and_placed_module_registers_its_middleware_and_media_signer() { let settings = placed(); assert!(validate(&settings).expect("should validate")); + let registration = register(&settings) + .expect("should register") + .expect("a selected module with a worker token should register"); + let names: Vec<&str> = registration + .middleware + .iter() + .map(|middleware| middleware.middleware_id()) + .collect(); + assert_eq!( + names, + [MODULE, STYLESHEET_MIDDLEWARE], + "should supply the page and stylesheet middleware" + ); + assert_eq!( + registration + .media_signer + .as_ref() + .map(|signer| signer.signer_id()), + Some(MODULE), + "should register the media signer alongside the text path" + ); + let stylesheet = ®istration.middleware[1]; + assert!(stylesheet.handles_media_type("text/css")); assert!( - register(&settings).expect("should register").is_some(), - "a selected module with a worker token should register" + stylesheet.handles_path("/assets/site.css"), + "the stylesheet path should not inherit the page path's asset exclusions" ); } @@ -138,10 +163,6 @@ fn required_settings_and_bounds_are_enforced() { "config without {field} should fail" ); } - let defaults = parsed(&config_value()).expect("should parse defaults"); - assert_eq!(defaults.policy_ttl_seconds, 300); - assert_eq!(defaults.plan_ttl_seconds, 604_800); - assert_eq!(defaults.max_hold_ms, 5_000); for (field, value) in [ ("api_base_url", json!("http://api.example.com")), ("api_base_url", json!("https://api.example.com/?q=1")), @@ -155,6 +176,17 @@ fn required_settings_and_bounds_are_enforced() { ("negative_ttl_seconds", json!(10)), ("policy_ttl_seconds", json!(301)), ("article_selector", json!("a".repeat(92))), + ("max_media_bytes", json!(100)), + ("max_media_bytes", json!(8_000_001)), + ("max_media_sign_calls_per_document", json!(0)), + ("max_media_sign_calls_per_document", json!(129)), + ("max_large_media_bytes", json!(1)), + ( + "max_large_media_bytes", + json!(MAX_PREPARED_SOURCE_BYTES + 1), + ), + ("max_large_media_ms", json!(4_999)), + ("max_large_media_ms", json!(600_001)), ("enabled", json!(true)), ("path_prefixes", json!(["/news/"])), ] { @@ -165,6 +197,20 @@ fn required_settings_and_bounds_are_enforced() { "{field} = {value} should be rejected" ); } + for value in [0, MIN_LARGE_MEDIA_BYTES, MAX_PREPARED_SOURCE_BYTES] { + let mut config = config_value(); + config["max_large_media_bytes"] = json!(value); + assert!( + parsed(&config).is_ok(), + "max_large_media_bytes = {value} should be accepted, 0 turning the lane off" + ); + } + for limit in [1, 64, 128] { + let mut config = config_value(); + config["max_media_sign_calls_per_document"] = json!(limit); + let accepted = parsed(&config).expect("bounded media-call limit"); + assert_eq!(accepted.max_media_sign_calls_per_document, limit); + } for (max_hold_ms, timeout_ms, valid) in [ (499, 250, false), (500, 250, true), @@ -184,6 +230,157 @@ fn required_settings_and_bounds_are_enforced() { } } +#[test] +fn article_selector_rejects_malformed_classes_at_settings_admission() { + for selector in [ + ".", + "..article", + ".123article", + ".article.body", + ".article body", + ".article>p", + ".article,main", + ".article#body", + ".article:hover", + ".article[data-body=yes]", + ".article\\:body", + ] { + let table = format!("{}article_selector = {}\n", table(), json!(selector)); + let settings = page_settings(&module_settings(&table, true)); + assert!( + validate(&settings).is_err(), + "malformed class selector {selector:?} should fail settings admission" + ); + assert!( + register(&settings).is_err(), + "malformed class selector {selector:?} should not register" + ); + } +} + +#[test] +fn article_selector_rejects_malformed_attributes_at_settings_admission() { + for selector in [ + "[", + "[itemprop]", + "[=articleBody]", + "[itemprop=]", + "[itemprop=articleBody", + "[itemprop=articleBody]p", + "[itemprop~=articleBody]", + "[itemprop=articleBody main]", + "[itemprop=articleBody=main]", + "[itemprop=[articleBody]]", + "[itemprop=\"articleBody']", + "[itemprop='articleBody' i]", + "[itemprop=articleBody][role=main]", + "[itemprop=articleBody],main", + "[itemprop='article\\Body']", + ] { + let table = format!("{}article_selector = {}\n", table(), json!(selector)); + let settings = page_settings(&module_settings(&table, true)); + assert!( + validate(&settings).is_err(), + "malformed attribute selector {selector:?} should fail settings admission" + ); + assert!( + register(&settings).is_err(), + "malformed attribute selector {selector:?} should not register" + ); + } +} + +#[test] +fn article_selector_rejects_malformed_tags_at_settings_admission() { + for selector in [ + " ", + " article", + "article ", + "123article", + "
", + "article.body", + "article p", + "article>p", + "article+p", + "article~p", + "article,main", + "article#body", + "article:hover", + "article[role=main]", + "#article", + "*", + ] { + let table = format!("{}article_selector = {}\n", table(), json!(selector)); + let settings = page_settings(&module_settings(&table, true)); + assert!( + validate(&settings).is_err(), + "malformed tag selector {selector:?} should fail settings admission" + ); + assert!( + register(&settings).is_err(), + "malformed tag selector {selector:?} should not register" + ); + } +} + +#[test] +fn article_selector_accepts_supported_syntax_and_selects_the_override() { + let selected = "Selected."; + let html = format!("
Fallback.
{selected}"); + for selector in [ + ".story-body", + "._story", + ".-story", + ".--story", + "[itemprop=articleBody]", + "[itemprop='articleBody']", + "[itemprop=\"articleBody\"]", + "[data-body='article body']", + "custom-story", + "CUSTOM-STORY", + ] { + let table = format!("{}article_selector = {}\n", table(), json!(selector)); + let settings = page_settings(&module_settings(&table, true)); + assert!( + validate(&settings).expect("should admit supported selector"), + "supported selector {selector:?} should select the module" + ); + assert!( + register(&settings) + .expect("should register supported selector") + .is_some(), + "supported selector {selector:?} should register" + ); + let boundary = boundary::find_article_boundary(&html, Some(selector)) + .expect("should find the configured override"); + assert_eq!( + boundary.selector, + format!("override:{selector}"), + "supported selector should take priority over the article fallback" + ); + assert_eq!(html.get(boundary.start..boundary.end), Some(selected)); + } +} + +#[test] +fn article_selector_empty_or_valid_but_absent_keeps_article_fallback() { + let selected = "

Fallback article.

"; + let html = format!("
Other region.
{selected}"); + for selector in ["", ".absent-story", "[data-body=absent]", "custom-story"] { + let table = format!("{}article_selector = {}\n", table(), json!(selector)); + let settings = page_settings(&module_settings(&table, true)); + assert!( + validate(&settings).expect("should admit empty or absent selector"), + "selector {selector:?} should select the module" + ); + let boundary = boundary::find_article_boundary(&html, Some(selector)) + .expect("should find the fallback article"); + assert_eq!(boundary.selector, "article"); + assert_eq!(html.get(boundary.start..boundary.end), Some(selected)); + } +} + #[test] fn excluded_paths_leave_the_middleware_out_of_the_chain() { let middleware = Encypher { @@ -312,8 +509,20 @@ fn served_response( request: Request, streaming: bool, ) -> http::Response { + try_served_response(settings, services, request, streaming).expect("should finish the page") +} + +fn try_served_response( + settings: &Settings, + services: &RuntimeServices, + request: Request, + streaming: bool, +) -> Result, error_stack::Report> { let registry = IntegrationRegistry::with_registrations(settings, &[builder()]) .expect("should build the registry"); + // As every adapter does, the request's services carry the registry's + // media signer. + let services = &services.clone().with_media_signer(registry.media_signer()); let plan = compile_auction_plan(settings).expect("should compile the auction plan"); let orchestrator = Arc::new(AuctionOrchestrator::from_plan(Arc::new(plan), None)); let method = request.method().clone(); @@ -338,8 +547,7 @@ fn served_response( request, EdgeCacheHeader::SurrogateControl, ) - .await - .expect("should fetch the page"); + .await?; if streaming { publisher_response_into_streaming_response( publisher, @@ -361,7 +569,6 @@ fn served_response( ) .await } - .expect("should finish the page") }) } @@ -427,6 +634,35 @@ fn the_publisher_path_signs_an_article_through_both_finalizers() { } } +#[test] +fn an_article_on_a_publisher_addressed_by_path_is_signed_for_its_own_address() { + let stub = stub("public, max-age=60"); + let mut request = page_request(Method::GET); + request + .extensions_mut() + .insert(trusted_server_core::addressing::RemovedPathPrefix::new( + "/www.site.example".to_owned(), + )); + + let page = served(&placed(), &services_for(&stub), request, false); + + assert!(page.contains(MARKER), "should sign the article: {page}"); + let policy_uri = &stub.recorded_request_uris()[1]; + assert!( + policy_uri.ends_with( + "page_url=http%3A%2F%2Fpublisher.example.com%2Fwww.site.example%2Fnews%2Fharbor-vote" + ), + "should ask the policy for the address the reader asked for: {policy_uri}" + ); + let sign = String::from_utf8(stub.recorded_request_bodies()[2].clone()).expect("JSON body"); + assert!( + sign.contains( + "\"page_url\":\"http://publisher.example.com/www.site.example/news/harbor-vote\"" + ), + "should sign the article for the address the reader asked for: {sign}" + ); +} + /// The unsigned bytes produced by the same public publisher pipeline before /// its final writer applies the configured bound. Core middleware may rewrite /// the origin body before Encypher sees it, so the origin fixture length is not @@ -521,3 +757,961 @@ fn a_page_the_origin_marked_private_is_never_sent() { "should call the origin and nothing else" ); } + +/// The pixels of a small image an origin embeds or serves. +const SOURCE_IMAGE: &[u8] = b"\x89PNG\r\n\x1a\n-source-pixels"; +/// The same image with an embedded C2PA manifest, as the media endpoint +/// answers it. +const SIGNED_IMAGE: &[u8] = b"\x89PNG\r\n\x1a\n-source-pixels-caBX-jumb-c2pa-manifest"; + +fn encoded(bytes: &[u8]) -> String { + base64::engine::general_purpose::STANDARD.encode(bytes) +} + +/// The headers of an embedded-manifest answer of the media endpoint for +/// `body` of `content_type`. +fn media_answer_as(body: &[u8], content_type: &str) -> Vec<(String, String)> { + [ + ("content-type", content_type.to_owned()), + ("x-encypher-policy-digest", POLICY_DIGEST.to_owned()), + ("x-encypher-signed-sha256", media_api::sha256_hex(body)), + ("x-encypher-manifest-instance-id", "xmp:iid:0001".to_owned()), + ("x-encypher-manifest-disposition", "embedded".to_owned()), + ("x-encypher-manifest-link-required", "false".to_owned()), + ( + "x-encypher-verification-url", + "https://verify.example.com/doc-2".to_owned(), + ), + ("x-encypher-document-id", "doc-2".to_owned()), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect() +} + +/// [`ARTICLE`] with a lazily loaded embedded image inside the article. +fn article_with_image() -> String { + ARTICLE.replace( + "
", + &format!( + "\"Harbor\"
", + encoded(SOURCE_IMAGE) + ), + ) +} + +/// An origin answering `body` as `content_type`, publicly cacheable. +fn origin_answer(stub: &StubHttpClient, body: &[u8], content_type: &str) { + stub.push_response_with_headers( + 200, + body.to_vec(), + vec![ + ("content-type", content_type), + ("cache-control", "public, max-age=60"), + ], + ); +} + +/// The policy answer the media lane asks for live, before it signs or reuses +/// embedded media. +fn live_policy(stub: &StubHttpClient) { + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); +} + +#[test] +fn a_revoked_credential_stops_embedded_media_reuse_while_the_cached_policy_lives() { + let kv = Arc::new(MemoryKv::default()); + let first = Arc::new(StubHttpClient::new()); + origin_answer( + &first, + article_with_image().as_bytes(), + "text/html; charset=utf-8", + ); + text_answers(&first); + live_policy(&first); + first.push_response_with_headers(200, SIGNED_IMAGE.to_vec(), media_answer(SIGNED_IMAGE)); + let warmed = served( + &placed(), + &services_for_kv(&first, Arc::clone(&kv)), + page_request(Method::GET), + false, + ); + assert!( + warmed.contains(&encoded(SIGNED_IMAGE)), + "should warm the policy and media caches" + ); + + // The cached policy and the cached signed image are both still fresh; + // the service now refuses the credential. + let revoked = Arc::new(StubHttpClient::new()); + origin_answer( + &revoked, + article_with_image().as_bytes(), + "text/html; charset=utf-8", + ); + revoked.push_response(403, b"{}".to_vec()); + let page = served( + &placed(), + &services_for_kv(&revoked, kv), + page_request(Method::GET), + false, + ); + + assert!( + !page.contains(&encoded(SIGNED_IMAGE)), + "should never reuse a signed image under a revoked credential" + ); + assert!( + page.contains(&encoded(SOURCE_IMAGE)), + "should deliver the image as the origin wrote it" + ); + assert!( + !revoked + .recorded_request_methods() + .iter() + .any(|method| method == "POST"), + "should sign nothing" + ); +} + +/// The policy and article-sign answers [`stub`] queues after the origin. +fn text_answers(stub: &StubHttpClient) { + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + stub.push_response( + 200, + serde_json::to_vec(&json!({ + "success": true, + "policy_digest": POLICY_DIGEST, + "embedding_plan": { + "index_unit": "codepoint", + "operations": [{ "insert_after_index": 5, "marker": MARKER }], + }, + "manifest_url": MANIFEST_URL, + })) + .expect("should encode"), + ); +} + +#[test] +fn the_publisher_path_signs_the_article_then_its_embedded_image() { + for streaming in [false, true] { + let stub = Arc::new(StubHttpClient::new()); + origin_answer( + &stub, + article_with_image().as_bytes(), + "text/html; charset=utf-8", + ); + text_answers(&stub); + live_policy(&stub); + stub.push_response_with_headers(200, SIGNED_IMAGE.to_vec(), media_answer(SIGNED_IMAGE)); + + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + streaming, + ); + + assert!( + page.contains(MARKER), + "streaming={streaming}: should sign the text" + ); + assert!( + page.contains(&format!( + "src=\"data:image/png;base64,{}\" alt=\"Harbor\"", + encoded(SIGNED_IMAGE) + )), + "streaming={streaming}: should deliver the signed image bytes in place: {page}" + ); + assert!( + !page.contains(&encoded(SOURCE_IMAGE)), + "streaming={streaming}: should leave no unsigned copy" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET", "GET", "POST", "GET", "POST"], + "streaming={streaming}: origin, policy, text sign, live policy, media sign" + ); + assert!( + stub.recorded_request_uris()[4].ends_with("/api/v1/public/cdn/media/sign"), + "streaming={streaming}: should call the media endpoint" + ); + let media_body = &stub.recorded_request_bodies()[4]; + assert!( + media_body + .windows(SOURCE_IMAGE.len()) + .any(|window| window == SOURCE_IMAGE), + "streaming={streaming}: should send the decoded source bytes" + ); + } +} + +#[test] +fn a_refused_embedded_image_stays_as_written_and_the_text_is_still_signed() { + let stub = Arc::new(StubHttpClient::new()); + origin_answer( + &stub, + article_with_image().as_bytes(), + "text/html; charset=utf-8", + ); + text_answers(&stub); + live_policy(&stub); + stub.push_response( + 422, + serde_json::to_vec(&json!({ + "success": false, + "error": {"code": "UNSUPPORTED_MEDIA_TYPE", "message": "x"}, + })) + .expect("should encode"), + ); + + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + false, + ); + + assert!(page.contains(MARKER), "should still sign the text"); + assert!( + page.contains(&encoded(SOURCE_IMAGE)), + "should leave the unsigned image as the origin wrote it" + ); + assert!( + !page.contains(&encoded(SIGNED_IMAGE)), + "should never present unsigned bytes as signed" + ); +} + +/// The module's settings with a stylesheet entry, scoped by `path` when one +/// is given. +fn stylesheet_settings(path: Option<&str>) -> String { + let path = path.map_or(String::new(), |path| format!("path = \"{path}\"\n")); + format!( + "{}\n[[serve]]\nmedia_type = \"text/css\"\n{path}middleware = [\"{STYLESHEET_MIDDLEWARE}\"]\n", + module_settings(&table(), true) + ) +} + +#[test] +fn a_stylesheet_entry_for_every_path_is_refused() { + for path in [None, Some("/")] { + let settings = page_settings(&stylesheet_settings(path)); + let error = validate(&settings).expect_err("should refuse an unscoped entry"); + assert!( + format!("{error:?}").contains("integrity"), + "{path:?}: should explain the integrity risk: {error:?}" + ); + } + let scoped = page_settings(&stylesheet_settings(Some("/assets/"))); + assert!(validate(&scoped).expect("should accept a scoped entry")); +} + +#[test] +fn the_publisher_path_signs_media_a_stylesheet_embeds() { + let settings = page_settings(&stylesheet_settings(Some("/assets/"))); + let stylesheet = format!( + ".hero{{background-image:url(\"data:image/png;base64,{}\")}}", + encoded(SOURCE_IMAGE) + ); + let stub = Arc::new(StubHttpClient::new()); + origin_answer(&stub, stylesheet.as_bytes(), "text/css; charset=utf-8"); + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + live_policy(&stub); + stub.push_response_with_headers(200, SIGNED_IMAGE.to_vec(), media_answer(SIGNED_IMAGE)); + let request = Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/assets/site.css") + .header(header::HOST, "publisher.example.com") + .body(EdgeBody::empty()) + .expect("should build the stylesheet request"); + + let served_stylesheet = served(&settings, &services_for(&stub), request, false); + + assert_eq!( + served_stylesheet, + format!( + ".hero{{background-image:url(\"data:image/png;base64,{}\")}}", + encoded(SIGNED_IMAGE) + ), + "should replace the embedded image with its signed bytes and nothing else" + ); +} + +/// The headers of an embedded-manifest PNG answer for `body`. +fn media_answer(body: &[u8]) -> Vec<(String, String)> { + media_answer_as(body, "image/png") +} + +fn media_request(path: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("https://publisher.example.com{path}")) + .header(header::HOST, "publisher.example.com") + .header(header::ACCEPT, "*/*") + .body(EdgeBody::empty()) + .expect("should build the media request") +} + +fn body_of(response: http::Response) -> bytes::Bytes { + block_on(response.into_body().into_bytes_bounded(16 * 1024 * 1024)) + .expect("should read the body") +} + +#[test] +fn the_publisher_path_delivers_the_signed_bytes_of_images_audio_and_video_on_any_path() { + for (path, content_type) in [ + ("/images/harbor.png", "image/png"), + ("/media/4711", "image/png"), + ("/video/ferry.mp4", "video/mp4"), + ("/audio/council.mp3", "audio/mpeg"), + ] { + let stub = Arc::new(StubHttpClient::new()); + origin_answer(&stub, SOURCE_IMAGE, content_type); + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + stub.push_response_with_headers( + 200, + SIGNED_IMAGE.to_vec(), + media_answer_as(SIGNED_IMAGE, content_type), + ); + + let response = served_response(&placed(), &services_for(&stub), media_request(path), false); + + assert_eq!(response.status(), 200, "{path}: should answer"); + let headers = response.headers().clone(); + assert_eq!( + headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some(content_type), + "{path}: should keep the media type" + ); + assert_eq!( + headers + .get("x-encypher-document-id") + .and_then(|value| value.to_str().ok()), + Some("doc-2"), + "{path}: should name the signed document" + ); + assert_eq!( + headers + .get(header::ETAG) + .and_then(|value| value.to_str().ok()), + Some(format!("\"sig-{}\"", media_api::sha256_hex(SIGNED_IMAGE)).as_str()), + "{path}: should validate the signed bytes, not the origin's" + ); + assert_eq!( + body_of(response).as_ref(), + SIGNED_IMAGE, + "{path}: should deliver the exact signed bytes" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET", "GET", "POST"], + "{path}: origin, policy, media sign" + ); + let metadata = &stub.recorded_request_bodies()[2]; + assert!( + metadata + .windows(SOURCE_IMAGE.len()) + .any(|window| window == SOURCE_IMAGE), + "{path}: should send the captured origin bytes" + ); + assert!( + metadata + .windows(b"\"origin_host\":\"".len()) + .any(|window| window == b"\"origin_host\":\""), + "{path}: should name the host core fetched from" + ); + } +} + +#[test] +fn a_failed_media_signature_delivers_the_origin_bytes_unmarked() { + let stub = Arc::new(StubHttpClient::new()); + origin_answer(&stub, SOURCE_IMAGE, "image/png"); + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + // An answer whose bytes are not the ones it names is never delivered. + stub.push_response_with_headers( + 200, + b"\x89PNG\r\n\x1a\nswapped-c2pa".to_vec(), + media_answer(SIGNED_IMAGE), + ); + + let response = served_response( + &placed(), + &services_for(&stub), + media_request("/images/harbor.png"), + false, + ); + + assert!( + response.headers().get("x-encypher-document-id").is_none(), + "should claim no signature" + ); + assert_eq!( + body_of(response).as_ref(), + SOURCE_IMAGE, + "should deliver the origin's bytes as they were" + ); +} + +#[test] +fn a_media_segment_passes_through_as_the_origin_sent_it() { + // A C2PA verifiable-segment-info segment is bound by its stream's init + // manifest. Signing it alone would fail, and the unsigned fallback would + // make it private and drop its validators. + const SEGMENT: &[u8] = b"\0\0\0\x18stypmsdh\0\0\0\0msdhmsixmoof-mdat-emsg"; + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + SEGMENT.to_vec(), + vec![ + ("content-type", "video/iso.segment"), + ("cache-control", "public, max-age=60"), + ("etag", "\"segment-7\""), + ], + ); + + let response = served_response( + &placed(), + &services_for(&stub), + media_request("/live/ferry/segment-7.m4s"), + false, + ); + + assert_eq!(response.status(), 200, "should answer"); + let headers = response.headers().clone(); + assert_eq!( + headers + .get(header::ETAG) + .and_then(|value| value.to_str().ok()), + Some("\"segment-7\""), + "should keep the origin's validator" + ); + assert!( + headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| !value.contains("private")), + "should not make the segment private" + ); + assert!( + headers.get("x-encypher-document-id").is_none(), + "should claim no signature" + ); + assert_eq!( + body_of(response).as_ref(), + SEGMENT, + "should deliver the segment byte for byte" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET"], + "should ask neither the policy nor the signing endpoint" + ); +} + +/// One ISO BMFF box of `kind` around `body`. +fn bmff_box(kind: &[u8; 4], body: &[u8]) -> Vec { + let size = u32::try_from(body.len() + 8).expect("should fit a 32-bit size"); + [&size.to_be_bytes()[..], kind, body].concat() +} + +#[test] +fn fragmented_stream_parts_typed_as_mp4_pass_through_as_the_origin_sent_them() { + let init = [ + bmff_box(b"ftyp", b"cmfc\0\0\0\0iso6cmfc"), + bmff_box( + b"moov", + &[ + bmff_box(b"mvhd", &[0; 100]), + bmff_box(b"mvex", &bmff_box(b"trex", &[0; 24])), + bmff_box(b"pssh", &[0; 32]), + ] + .concat(), + ), + ] + .concat(); + let segment = [ + bmff_box(b"styp", b"msdh\0\0\0\0msdhmsix"), + bmff_box(b"emsg", b"\0\0\0\0urn:c2pa:verifiable-segment-info\0fseg\0"), + bmff_box(b"sidx", &[0; 32]), + bmff_box(b"moof", &bmff_box(b"mfhd", &[0; 8])), + bmff_box(b"mdat", b"coded-samples"), + ] + .concat(); + for (path, part) in [ + ("/live/ferry/init.mp4", init), + ("/live/ferry/7.m4s", segment), + ] { + // No `no-transform`: the bytes alone keep the part from being signed. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + part.clone(), + vec![ + ("content-type", "video/mp4"), + ("cache-control", "public, max-age=60"), + ("etag", "\"part-7\""), + ], + ); + + let response = served_response(&placed(), &services_for(&stub), media_request(path), false); + + assert_eq!(response.status(), 200, "{path}: should answer"); + let headers = response.headers().clone(); + assert_eq!( + headers + .get(header::ETAG) + .and_then(|value| value.to_str().ok()), + Some("\"part-7\""), + "{path}: should keep the origin's validator" + ); + assert!( + headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| !value.contains("private")), + "{path}: should not make the part private" + ); + assert!( + headers.get(header::LINK).is_none() && headers.get("x-encypher-document-id").is_none(), + "{path}: should add no manifest" + ); + assert_eq!( + body_of(response).as_ref(), + part.as_slice(), + "{path}: should deliver the part byte for byte" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET"], + "{path}: should ask neither the policy nor the signing endpoint" + ); + } +} + +#[test] +fn a_progressive_mp4_is_still_handed_to_the_signing_endpoint() { + let progressive = [ + bmff_box(b"ftyp", b"isom\0\0\0\0isomavc1"), + bmff_box(b"moov", &bmff_box(b"trak", &[0; 16])), + bmff_box(b"mdat", b"coded-samples"), + ] + .concat(); + let stub = Arc::new(StubHttpClient::new()); + origin_answer(&stub, &progressive, "video/mp4"); + live_policy(&stub); + stub.push_response_with_headers( + 200, + SIGNED_IMAGE.to_vec(), + media_answer_as(SIGNED_IMAGE, "video/mp4"), + ); + + let _response = served_response( + &placed(), + &services_for(&stub), + media_request("/video/ferry.mp4"), + false, + ); + + assert_eq!( + stub.recorded_request_methods(), + ["GET", "GET", "POST"], + "origin, policy, media sign" + ); + let sent = &stub.recorded_request_bodies()[2]; + assert!( + sent.windows(progressive.len()) + .any(|window| window == progressive.as_slice()), + "should send the progressive file's exact bytes" + ); +} + +/// Signed source bytes deliberately contain targets of core's attribute and +/// text URL replacement before the provenance carrier/reference appears. +fn protected_svg(reference: bool) -> String { + let provenance = if reference { + "" + } else { + "QUJD" + }; + format!( + "\ + \ + https://origin.example.com/\ + \ + \ + {}{provenance}", + encoded(SOURCE_IMAGE), + "origin.example.com ".repeat(1024), + ) +} + +fn article_with_protected_svg(svg: &str) -> String { + ARTICLE.replace( + "", + &format!( + "{svg}\ + https://origin.example.com/outside" + ), + ) +} + +fn assert_source_and_surroundings(page: &str, svg: &str) { + assert!( + page.contains(svg), + "must retain the complete original SVG byte slice" + ); + assert!( + page.contains("http://publisher.example.com/outside"), + "core's ordinary outside URL edits must still run: {page}" + ); + assert!(page.contains(MARKER), "article signing must still run"); + assert!( + !page.contains("` and `` are whole comments, and `--!>` also ends one. -fn comment_end(bytes: &[u8], start: usize) -> Option { +pub(crate) fn comment_end(bytes: &[u8], start: usize) -> Option { let body = start + 4; if starts_with_at(bytes, body, b">") { return Some(body + 1); @@ -600,7 +600,7 @@ enum TagState { /// Returns the offset of the `>` ending the tag whose name starts at `start`. /// As in the HTML tokenizer, a quote delimits only a value it begins. -fn tag_end(bytes: &[u8], start: usize) -> Option { +pub(crate) fn tag_end(bytes: &[u8], start: usize) -> Option { let mut state = TagState::Name; for (offset, byte) in bytes.get(start..)?.iter().copied().enumerate() { let space = byte.is_ascii_whitespace(); @@ -696,7 +696,7 @@ pub(crate) fn is_js_whitespace(ch: char) -> bool { ) } -fn find_byte(bytes: &[u8], needle: u8, start: usize) -> Option { +pub(crate) fn find_byte(bytes: &[u8], needle: u8, start: usize) -> Option { let relative = bytes .get(start..)? .iter() @@ -704,7 +704,7 @@ fn find_byte(bytes: &[u8], needle: u8, start: usize) -> Option { Some(start.saturating_add(relative)) } -fn find_bytes(bytes: &[u8], needle: &[u8], start: usize) -> Option { +pub(crate) fn find_bytes(bytes: &[u8], needle: &[u8], start: usize) -> Option { if needle.is_empty() { return Some(start.min(bytes.len())); } @@ -715,7 +715,7 @@ fn find_bytes(bytes: &[u8], needle: &[u8], start: usize) -> Option { Some(start.saturating_add(relative)) } -fn starts_with_at(bytes: &[u8], start: usize, needle: &[u8]) -> bool { +pub(crate) fn starts_with_at(bytes: &[u8], start: usize, needle: &[u8]) -> bool { bytes .get(start..) .is_some_and(|tail| tail.starts_with(needle)) @@ -742,7 +742,7 @@ fn structural_tag<'a>(start: usize, token: &'a str, mode: ScanMode) -> Option bool { +pub(crate) fn is_structural_element_start(bytes: &[u8], start: usize) -> bool { match bytes.get(start + 1).copied() { Some(byte) if byte.is_ascii_alphabetic() => true, Some(b'/') => bytes.get(start + 2).is_some_and(u8::is_ascii_alphabetic), @@ -750,7 +750,7 @@ fn is_structural_element_start(bytes: &[u8], start: usize) -> bool { } } -fn raw_close_name_end(bytes: &[u8], start: usize, expected: &str) -> Option { +pub(crate) fn raw_close_name_end(bytes: &[u8], start: usize, expected: &str) -> Option { if !starts_with_at(bytes, start, b" Option Option<&str> { +pub(crate) fn take_tag_name(content: &str) -> Option<&str> { let end = content .char_indices() .find(|(_, ch)| ch.is_ascii_whitespace() || matches!(*ch, '/' | '>')) diff --git a/crates/shared/aws/src/lib.rs b/crates/shared/aws/src/lib.rs index e0210a1c6a..a70259e77a 100644 --- a/crates/shared/aws/src/lib.rs +++ b/crates/shared/aws/src/lib.rs @@ -371,6 +371,7 @@ mod tests { handle_asset_proxy_request( &settings, &services, + ®istry, request, &settings.proxy.asset_routes[0], source.as_ref(), diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index 5075f56b70..60b58f6246 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -21,6 +21,8 @@ path = "src/main.rs" [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } +# The dev server's own front end hands streamed bodies to Hyper (src/server.rs). +axum = { workspace = true } bytes = { workspace = true } edgezero-adapter-axum = { workspace = true, features = ["axum"] } edgezero-core = { workspace = true } @@ -29,7 +31,7 @@ futures = { workspace = true } log = { workspace = true } reqwest = { workspace = true } simple_logger = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "sync", "time"] } trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } @@ -63,7 +65,6 @@ trusted-server-testing-testlight = { workspace = true } [dev-dependencies] trusted-server-core = { workspace = true, features = ["test-utils"] } -axum = { workspace = true } base64 = { workspace = true } serde_json = { workspace = true } temp-env = { workspace = true } diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 1f13770b4e..819b76a878 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -433,7 +433,9 @@ impl AppState { if let Some(module) = self.registry.device_module() { services = services.with_device_module(module); } - services = services.with_identifier_modules(self.registry.identifier_modules()); + services = services + .with_identifier_modules(self.registry.identifier_modules()) + .with_media_signer(self.registry.media_signer()); // The publisher this request resolved to names every key in a shared // store and every name the browser holds. let prefixes = self.registry.prefixes( @@ -962,8 +964,15 @@ async fn dispatch_asset_fallback( let asset_route = &state.settings.proxy.asset_routes[index]; let answer = match state.registry.route_source(&state.settings, index) { Ok(source) => { - handle_asset_proxy_request(&state.settings, services, req, asset_route, source.as_ref()) - .await + handle_asset_proxy_request( + &state.settings, + services, + &state.registry, + req, + asset_route, + source.as_ref(), + ) + .await } Err(report) => Err(report), }; diff --git a/crates/trusted-server-adapter-axum/src/lib.rs b/crates/trusted-server-adapter-axum/src/lib.rs index 2f15e566d2..4dacd424e3 100644 --- a/crates/trusted-server-adapter-axum/src/lib.rs +++ b/crates/trusted-server-adapter-axum/src/lib.rs @@ -10,3 +10,5 @@ pub mod app; pub mod middleware; /// Platform-trait implementations backed by env vars and `reqwest`. pub mod platform; +/// HTTP front end that delivers streamed responses chunk by chunk. +pub mod server; diff --git a/crates/trusted-server-adapter-axum/src/main.rs b/crates/trusted-server-adapter-axum/src/main.rs index 0a88091dda..bdfec9b1d6 100644 --- a/crates/trusted-server-adapter-axum/src/main.rs +++ b/crates/trusted-server-adapter-axum/src/main.rs @@ -1,10 +1,11 @@ use std::path::Path; -use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; +use edgezero_adapter_axum::dev_server::AxumDevServerConfig; use edgezero_core::app::Hooks as _; use log::LevelFilter; use trusted_server_adapter_axum::app::TrustedServerApp; use trusted_server_adapter_axum::platform::{KV_PATH_ENV, open_kv_handle}; +use trusted_server_adapter_axum::server; /// Environment variable that sets the maximum log level. const LOG_LEVEL_ENV: &str = "RUST_LOG"; @@ -42,13 +43,13 @@ fn main() { log::info!("Listening on http://{}", config.addr); let router = TrustedServerApp::routes(); - let mut server = AxumDevServer::with_config(router, config); + let mut kv_handle = None; if let Some(path) = std::env::var_os(KV_PATH_ENV).filter(|path| !path.is_empty()) { // Construct the store once so every request shares one handle. match open_kv_handle(Path::new(&path)) { Ok(handle) => { log::info!("Runtime KV: {}", Path::new(&path).display()); - server = server.with_kv_handle(handle); + kv_handle = Some(handle); } Err(err) => { log::error!("{KV_PATH_ENV} is set but the KV store cannot be opened: {err}"); @@ -56,7 +57,9 @@ fn main() { } } } - if let Err(err) = server.run() { + // The adapter's own front end replaces `AxumDevServer`, whose service + // collects every streamed response into one buffer before sending it. + if let Err(err) = server::run(router, &config, kv_handle) { log::error!("trusted-server-adapter-axum failed: {err}"); std::process::exit(1); } diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 6736e71f23..9ca2bc5a9c 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -311,10 +311,17 @@ impl Future for AxumPendingHandle { /// reqwest-backed HTTP client for the Axum dev server. /// +/// [`send`](PlatformHttpClient::send) preserves the origin body as a +/// `Body::Stream` when the request asks for it: each chunk is read from the +/// connection only when the caller asks for the next one, so a large origin +/// response never sits whole in memory. The client's 30 second timeout still +/// covers the whole exchange, body included. +/// /// `send_async` buffers any `Body::Stream` in the calling context, then spawns /// a `tokio` task for each outbound request so that multiple `send_async` calls /// run concurrently. `select` uses [`futures::future::select_all`] to wait for -/// the first completing handle, preserving fan-out semantics. +/// the first completing handle, preserving fan-out semantics. That path stays +/// buffered, so it refuses a request that asks for a streamed response. pub struct AxumPlatformHttpClient { client: reqwest::Client, } @@ -436,12 +443,59 @@ impl AxumPlatformHttpClient { Ok(body) } + /// Yield the origin body chunk by chunk, enforcing `limit` before each + /// chunk is passed on. + /// + /// The chunk that would carry the total past `limit` becomes an error + /// and ends the stream, as does a failed read, so a truncated or + /// oversized origin body never reaches the caller as a complete one. + fn streamed_response_body( + response: reqwest::Response, + limit: Option, + ) -> edgezero_core::body::Body { + let stream = futures::stream::unfold(Some((response, 0usize)), move |state| async move { + let (mut response, received) = state?; + match response.chunk().await { + Ok(None) => None, + Ok(Some(chunk)) => { + let Some(next_received) = received.checked_add(chunk.len()) else { + return Some(( + Err(std::io::Error::other( + "upstream response body byte count overflowed", + )), + None, + )); + }; + if let Some(limit) = limit + && next_received > limit + { + return Some(( + Err(std::io::Error::other(format!( + "upstream response body exceeded caller limit of {limit} bytes" + ))), + None, + )); + } + Some((Ok(chunk), Some((response, next_received)))) + } + Err(error) => Some(( + Err(std::io::Error::other(format!( + "failed to read upstream response body: {error}" + ))), + None, + )), + } + }); + edgezero_core::body::Body::from_stream(stream) + } + async fn execute( &self, request: PlatformHttpRequest, ) -> Result> { Self::refuse_image_optimizer_metadata(&request)?; let max_response_body_bytes = request.max_response_body_bytes; + let stream_response = request.stream_response; let request_is_head = request.request.method() == edgezero_core::http::Method::HEAD; let uri = request.request.uri().to_string(); let method = reqwest::Method::from_bytes(request.request.method().as_str().as_bytes()) @@ -464,21 +518,40 @@ impl AxumPlatformHttpClient { .change_context(PlatformError::HttpClient) .attach(format!("outbound request to {uri} failed"))?; - let status = resp.status().as_u16(); - let mut edge_builder = edgezero_core::http::response_builder().status(status); + let status = resp.status(); + let mut edge_builder = edgezero_core::http::response_builder().status(status.as_u16()); for (name, value) in sanitized_response_headers(resp.headers()) { edge_builder = edge_builder.header(name.as_str(), value.as_slice()); } - let resp_bytes = - Self::response_body(resp, max_response_body_bytes, request_is_head).await?; - // A caller limit is enforced chunk by chunk before this Vec grows past it. - // Unbounded callers retain the dev server's previous whole-body behavior. - log::debug!( - "buffered {} upstream response bytes from {uri}", - resp_bytes.len() - ); + let body = if !stream_response { + let resp_bytes = + Self::response_body(resp, max_response_body_bytes, request_is_head).await?; + // A caller limit is enforced chunk by chunk before this Vec grows past it. + // Unbounded callers retain the dev server's previous whole-body behavior. + log::debug!( + "buffered {} upstream response bytes from {uri}", + resp_bytes.len() + ); + edgezero_core::body::Body::from(resp_bytes) + } else if !Self::response_carries_body(request_is_head, status) { + edgezero_core::body::Body::empty() + } else { + // Content-Length permits an early refusal. The stream enforces the + // same limit for missing or dishonest lengths. + if let Some(limit) = max_response_body_bytes + && resp + .content_length() + .is_some_and(|length| u64::try_from(limit).is_ok_and(|limit| length > limit)) + { + return Err(Report::new(PlatformError::HttpClient).attach(format!( + "upstream Content-Length exceeds caller response body limit of {limit} bytes" + ))); + } + log::debug!("streaming upstream response body from {uri}"); + Self::streamed_response_body(resp, max_response_body_bytes) + }; let edge_resp = edge_builder - .body(edgezero_core::body::Body::from(resp_bytes)) + .body(body) .change_context(PlatformError::HttpClient)?; Ok(PlatformResponse::new(edge_resp).with_backend_name(request.backend_name)) @@ -493,6 +566,10 @@ impl Default for AxumPlatformHttpClient { #[async_trait(?Send)] impl PlatformHttpClient for AxumPlatformHttpClient { + fn supports_streaming_responses(&self) -> bool { + true + } + async fn send( &self, request: PlatformHttpRequest, @@ -505,6 +582,11 @@ impl PlatformHttpClient for AxumPlatformHttpClient { request: PlatformHttpRequest, ) -> Result> { Self::refuse_image_optimizer_metadata(&request)?; + if request.stream_response { + return Err(Report::new(PlatformError::HttpClient).attach( + "the Axum dev server streams responses through send alone; send_async buffers", + )); + } let backend_name = request.backend_name.clone(); let max_response_body_bytes = request.max_response_body_bytes; @@ -900,6 +982,7 @@ mod tests { &Arc::default(), ) } + use futures::StreamExt as _; use std::time::Duration; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; @@ -1257,6 +1340,306 @@ mod tests { } } + /// Serve one connection: read the request through `request_end`, answer + /// `head_and_first`, wait for `release`, then send `rest`. The request + /// bytes the client sent come back through the returned receiver. + async fn serve_gated_response( + request_end: &'static [u8], + head_and_first: &'static [u8], + release: tokio::sync::oneshot::Receiver<()>, + rest: &'static [u8], + ) -> (String, tokio::sync::oneshot::Receiver>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("should bind gated HTTP test server"); + let addr = listener.local_addr().expect("should read local address"); + let (request_sender, request_receiver) = tokio::sync::oneshot::channel(); + + tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("should accept request"); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + while !request.ends_with(request_end) { + let read = stream.read(&mut buffer).await.expect("should read request"); + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + } + drop(request_sender.send(request)); + stream + .write_all(head_and_first) + .await + .expect("should write the head and first chunk"); + stream.flush().await.expect("should flush the first chunk"); + if release.await.is_ok() { + stream + .write_all(rest) + .await + .expect("should write the rest of the body"); + } + }); + + (format!("http://{addr}/media"), request_receiver) + } + + async fn next_chunk( + stream: &mut futures::stream::LocalBoxStream<'static, Result>, + ) -> Option> { + tokio::time::timeout(Duration::from_secs(5), stream.next()) + .await + .expect("the next chunk or the end should arrive promptly") + } + + #[test] + fn the_client_streams_through_send_and_not_through_send_async() { + let client = AxumPlatformHttpClient::new(); + assert!( + client.supports_streaming_responses(), + "send should preserve origin bodies as streams" + ); + assert!( + !client.supports_pending_streaming_responses(), + "send_async stays buffered" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_streamed_send_returns_before_the_origin_finishes_and_keeps_the_request() { + let (release, release_receiver) = tokio::sync::oneshot::channel(); + let (url, request_bytes) = serve_gated_response( + b"payload", + b"HTTP/1.1 200 OK\r\n\ + Content-Type: video/mp4\r\n\ + Transfer-Encoding: chunked\r\n\ + \r\n\ + 6\r\n\ + first-\r\n", + release_receiver, + b"6\r\nsecond\r\n0\r\n\r\n", + ) + .await; + let request = edgezero_core::http::request_builder() + .method("POST") + .uri(url) + .header(header::AUTHORIZATION, "Bearer example-token") + .body(EdgeBody::from(b"payload".to_vec())) + .expect("should build outbound request"); + + let response = tokio::time::timeout( + Duration::from_secs(5), + AxumPlatformHttpClient::new().send( + PlatformHttpRequest::new(request, "test_backend") + .with_stream_response() + .with_max_response_body_bytes(64), + ), + ) + .await + .expect("send should return while the origin body is still open") + .expect("the streamed send should succeed") + .response; + + let sent = String::from_utf8(request_bytes.await.expect("should capture the request")) + .expect("the request head should be text"); + assert!( + sent.starts_with("POST /media "), + "should keep the method: {sent}" + ); + assert!( + sent.to_ascii_lowercase() + .contains("authorization: bearer example-token"), + "should forward the authorization header: {sent}" + ); + assert!( + sent.ends_with("payload"), + "should forward the request body: {sent}" + ); + assert_eq!( + response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("video/mp4") + ); + let mut stream = response + .into_body() + .into_stream() + .expect("a streamed send should return a streamed body"); + let first = next_chunk(&mut stream) + .await + .expect("the first chunk should arrive before the origin finishes") + .expect("the first chunk should be readable"); + assert_eq!(first.as_ref(), b"first-"); + + release + .send(()) + .expect("should release the rest of the body"); + let mut rest = Vec::new(); + while let Some(chunk) = next_chunk(&mut stream).await { + rest.extend_from_slice(&chunk.expect("the rest of the body should be readable")); + } + assert_eq!(rest, b"second"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_streamed_send_ends_with_an_error_at_the_chunk_that_crosses_the_caller_limit() { + let url = serve_raw_response( + b"HTTP/1.1 200 OK\r\n\ + Transfer-Encoding: chunked\r\n\ + \r\n\ + 2\r\n\ + ab\r\n\ + 2\r\n\ + cd\r\n\ + 1\r\n\ + e\r\n\ + 0\r\n\ + \r\n", + ) + .await; + let request = edgezero_core::http::request_builder() + .uri(url) + .body(EdgeBody::empty()) + .expect("should build outbound request"); + + let mut stream = AxumPlatformHttpClient::new() + .send( + PlatformHttpRequest::new(request, "test_backend") + .with_stream_response() + .with_max_response_body_bytes(4), + ) + .await + .expect("a chunked response has no length to refuse up front") + .response + .into_body() + .into_stream() + .expect("a streamed send should return a streamed body"); + + let mut delivered = Vec::new(); + let error = loop { + match next_chunk(&mut stream).await { + Some(Ok(chunk)) => delivered.extend_from_slice(&chunk), + Some(Err(error)) => break Some(error), + None => break None, + } + } + .expect("a body over the limit must not end cleanly"); + assert!( + delivered.len() <= 4 && b"abcde".starts_with(&delivered), + "should pass on only bytes within the limit: {delivered:?}" + ); + assert!( + error.to_string().contains("limit of 4 bytes"), + "should report the enforced limit: {error}" + ); + assert!( + next_chunk(&mut stream).await.is_none(), + "should end at the error" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_streamed_send_refuses_a_declared_length_over_the_caller_limit() { + let url = + serve_raw_response(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n0123456789").await; + let request = edgezero_core::http::request_builder() + .uri(url) + .body(EdgeBody::empty()) + .expect("should build outbound request"); + + let result = AxumPlatformHttpClient::new() + .send( + PlatformHttpRequest::new(request, "test_backend") + .with_stream_response() + .with_max_response_body_bytes(4), + ) + .await; + + assert!( + result.is_err(), + "a declared length over the limit should be refused before any byte is read" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_streamed_send_passes_on_an_origin_that_closes_mid_body() { + // The origin declares ten bytes, sends four, and closes. + let url = serve_raw_response(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nabcd").await; + let request = edgezero_core::http::request_builder() + .uri(url) + .body(EdgeBody::empty()) + .expect("should build outbound request"); + + let mut stream = AxumPlatformHttpClient::new() + .send(PlatformHttpRequest::new(request, "test_backend").with_stream_response()) + .await + .expect("the head arrives before the origin closes") + .response + .into_body() + .into_stream() + .expect("a streamed send should return a streamed body"); + + let mut delivered = Vec::new(); + let outcome = loop { + match next_chunk(&mut stream).await { + Some(Ok(chunk)) => delivered.extend_from_slice(&chunk), + Some(Err(error)) => break Err(error), + None => break Ok(()), + } + }; + assert!( + outcome.is_err(), + "a truncated origin body must end with an error, after {delivered:?}" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_streamed_head_request_has_an_empty_body_whatever_its_declared_length() { + let url = serve_raw_response(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n").await; + let request = edgezero_core::http::request_builder() + .method("HEAD") + .uri(url) + .body(EdgeBody::empty()) + .expect("should build outbound request"); + + let response = AxumPlatformHttpClient::new() + .send( + PlatformHttpRequest::new(request, "test_backend") + .with_stream_response() + .with_max_response_body_bytes(4), + ) + .await + .expect("a HEAD answer carries no body for the limit to refuse") + .response; + + assert_eq!( + response + .into_body() + .into_bytes() + .expect("a bodiless answer should be an empty buffer") + .len(), + 0 + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn send_async_refuses_a_streamed_response_rather_than_buffer_it() { + let url = serve_raw_response(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok").await; + let request = edgezero_core::http::request_builder() + .uri(url) + .body(EdgeBody::empty()) + .expect("should build outbound request"); + + let result = AxumPlatformHttpClient::new() + .send_async(PlatformHttpRequest::new(request, "test_backend").with_stream_response()) + .await; + + assert!( + result.is_err(), + "the buffered send_async path must refuse a streamed response" + ); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn select_attributes_failed_backend_name() { // Bind and immediately drop a listener so the port is closed — the diff --git a/crates/trusted-server-adapter-axum/src/server.rs b/crates/trusted-server-adapter-axum/src/server.rs new file mode 100644 index 0000000000..e798a4f463 --- /dev/null +++ b/crates/trusted-server-adapter-axum/src/server.rs @@ -0,0 +1,468 @@ +//! The dev server's HTTP front end, which delivers streamed responses as +//! they are produced. +//! +//! `EdgeZero`'s own Axum service collects every `Body::Stream` response into +//! one buffer before Hyper sees it, so a large origin response streamed +//! through the platform client would be held whole in memory on its way to +//! the client. This front end dispatches through the same `EdgeZero` router +//! and request conversion, and hands a streamed body to Hyper chunk by +//! chunk instead. +//! +//! The router's futures and a streamed body are `!Send`, while Hyper needs a +//! `Send` body. Each request therefore runs on a Tokio blocking-pool thread, +//! which drives the router through the runtime handle, as `EdgeZero`'s +//! service does with `block_in_place`. That thread then passes the body's +//! chunks to Hyper through a bounded channel, so a slow client holds back +//! the origin rather than letting chunks pile up. A failed chunk is passed on +//! as a body error, which makes Hyper abort the response instead of ending +//! it as though it were complete. A client that leaves closes the channel, +//! which stops the router or the body read at once and drops the origin +//! request. +//! +//! At most [`MAX_CONCURRENT_REQUESTS`] requests hold a blocking thread at a +//! time. Later requests wait for a permit in their own connection task, so +//! no work is queued behind the blocking pool without a bound. + +use std::net::SocketAddr; +use std::sync::Arc; + +use axum::body::Body as AxumBody; +use axum::extract::Request as AxumRequest; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::Response as AxumResponse; +use bytes::Bytes; +use edgezero_adapter_axum::dev_server::AxumDevServerConfig; +use edgezero_adapter_axum::request::into_core_request; +use edgezero_core::body::Body as EdgeBody; +use edgezero_core::key_value_store::KvHandle; +use edgezero_core::router::RouterService; +use edgezero_core::store_registry::KvRegistry; +use futures::StreamExt as _; +use tokio::sync::{Semaphore, mpsc, oneshot}; + +/// Chunks a streamed body may hold between the router and Hyper before the +/// producer waits for the client. +const STREAMED_CHUNKS_IN_FLIGHT: usize = 8; + +/// Requests that may hold a blocking-pool thread at once. +/// +/// Half of Tokio's default 512 blocking threads, so the KV store's own +/// blocking operations, which a request waits on, always find a thread. +const MAX_CONCURRENT_REQUESTS: usize = 256; + +/// The store id `EdgeZero` binds a single KV handle under. +const DEFAULT_KV_STORE_ID: &str = "default"; + +/// Run the dev server on its own multi-threaded Tokio runtime. +/// +/// `kv_handle` is bound as the `default` KV store of every request, as +/// `EdgeZero`'s dev server binds it. +/// +/// # Errors +/// +/// Returns an error when the runtime cannot start, the address cannot be +/// bound, or the server's accept loop fails. +pub fn run( + router: RouterService, + config: &AxumDevServerConfig, + kv_handle: Option, +) -> std::io::Result<()> { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + let addr = config.addr; + let enable_ctrl_c = config.enable_ctrl_c; + runtime.block_on(async move { + let listener = tokio::net::TcpListener::bind(addr).await.map_err(|error| { + std::io::Error::new( + error.kind(), + format!("failed to bind dev server to {addr}: {error}"), + ) + })?; + let kv_registry = + kv_handle.map(|handle| KvRegistry::single_id(DEFAULT_KV_STORE_ID.to_owned(), handle)); + if enable_ctrl_c { + serve(listener, router, kv_registry, async { + if let Err(error) = tokio::signal::ctrl_c().await { + log::error!("failed to listen for ctrl-c: {error}"); + std::future::pending::<()>().await; + } + }) + .await + } else { + serve(listener, router, kv_registry, std::future::pending()).await + } + }) +} + +/// Serve requests from `listener` until `shutdown` completes. +async fn serve( + listener: tokio::net::TcpListener, + router: RouterService, + kv_registry: Option, + shutdown: F, +) -> std::io::Result<()> +where + F: Future + Send + 'static, +{ + let permits = Arc::new(Semaphore::new(MAX_CONCURRENT_REQUESTS)); + let app = axum::Router::new().fallback(move |request: AxumRequest| { + dispatch( + router.clone(), + kv_registry.clone(), + Arc::clone(&permits), + request, + ) + }); + axum::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(shutdown) + .await +} + +/// Answer one request, returning as soon as the response head is ready. +async fn dispatch( + router: RouterService, + kv_registry: Option, + permits: Arc, + request: AxumRequest, +) -> AxumResponse { + let Ok(permit) = permits.acquire_owned().await else { + return plain_response( + StatusCode::SERVICE_UNAVAILABLE, + "server is shutting down".to_owned(), + ); + }; + let (head_sender, head) = oneshot::channel(); + let runtime = tokio::runtime::Handle::current(); + let worker = tokio::task::spawn_blocking(move || { + runtime.block_on(produce(router, kv_registry, request, head_sender)); + drop(permit); + }); + match head.await { + Ok(response) => response, + Err(_) => { + let reason = match worker.await { + Err(error) => format!("request worker failed: {error}"), + Ok(()) => "request worker ended without a response".to_owned(), + }; + log::error!("{reason}"); + plain_response(StatusCode::INTERNAL_SERVER_ERROR, reason) + } + } +} + +/// Run the router for `request`, send the response head, then feed a +/// streamed body to Hyper until it ends, fails, or the client leaves. +/// +/// The error answers match `EdgeZero`'s Axum service: a request that cannot +/// be converted answers 500 with the conversion error, and a router error +/// answers 500 with `internal error: `. +async fn produce( + router: RouterService, + kv_registry: Option, + request: AxumRequest, + mut head: oneshot::Sender, +) { + let mut core_request = match into_core_request(request).await { + Ok(core_request) => core_request, + Err(error) => { + drop(head.send(plain_response(StatusCode::INTERNAL_SERVER_ERROR, error))); + return; + } + }; + if let Some(registry) = kv_registry { + core_request.extensions_mut().insert(registry); + } + let routed = tokio::select! { + routed = router.oneshot(core_request) => routed, + // Hyper dropped the request future: the client left before the head. + () = head.closed() => return, + }; + let core_response = match routed { + Ok(core_response) => core_response, + Err(error) => { + drop(head.send(plain_response( + StatusCode::INTERNAL_SERVER_ERROR, + format!("internal error: {error}"), + ))); + return; + } + }; + + let (parts, body) = core_response.into_parts(); + let mut stream = match body { + EdgeBody::Once(bytes) => { + drop(head.send(AxumResponse::from_parts(parts, AxumBody::from(bytes)))); + return; + } + EdgeBody::Stream(stream) => stream, + }; + + let (chunk_sender, chunk_receiver) = + mpsc::channel::>(STREAMED_CHUNKS_IN_FLIGHT); + let body = AxumBody::from_stream(futures::stream::unfold( + chunk_receiver, + |mut receiver| async move { receiver.recv().await.map(|chunk| (chunk, receiver)) }, + )); + if head.send(AxumResponse::from_parts(parts, body)).is_err() { + return; + } + loop { + let next = tokio::select! { + next = stream.next() => next, + // Hyper dropped the body: the client left mid-stream. + () = chunk_sender.closed() => return, + }; + let Some(chunk) = next else { + return; + }; + let failed = chunk.is_err(); + let chunk = chunk.map_err(|error| { + log::warn!("streamed response failed mid-body: {error:#}"); + std::io::Error::other(format!("{error:#}")) + }); + // A send waits while the channel is full, so a slow client slows + // the origin read. It fails once Hyper has dropped the body. + if chunk_sender.send(chunk).await.is_err() || failed { + return; + } + } +} + +fn plain_response(status: StatusCode, message: String) -> AxumResponse { + let mut response = AxumResponse::new(AxumBody::from(message)); + *response.status_mut() = status; + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + response +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + use edgezero_core::context::RequestContext; + use edgezero_core::error::EdgeError; + + /// Start the front end on a free local port and return its base URL. + async fn start(router: RouterService) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("should bind a local port"); + let addr = listener + .local_addr() + .expect("should read the bound address"); + tokio::spawn(serve(listener, router, None, std::future::pending())); + format!("http://{addr}") + } + + /// A body that yields `first`, waits for `release`, then yields `second`. + fn gated_body( + first: &'static [u8], + release: Arc, + second: Result<&'static [u8], &'static str>, + ) -> EdgeBody { + let stream = futures::stream::unfold(0u8, move |step| { + let release = Arc::clone(&release); + async move { + match step { + 0 => Some((Ok(Bytes::from_static(first)), 1)), + 1 => { + release.notified().await; + let item = second + .map(Bytes::from_static) + .map_err(std::io::Error::other); + Some((item, 2)) + } + _ => None, + } + } + }); + EdgeBody::from_stream(stream) + } + + /// Records that the body it travels with was dropped. + struct DropFlag(Arc); + + impl Drop for DropFlag { + fn drop(&mut self) { + self.0.store(true, std::sync::atomic::Ordering::SeqCst); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_client_that_leaves_stops_the_body_read() { + let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let handler_dropped = Arc::clone(&dropped); + let router = RouterService::builder() + .get("/media", move |_ctx: RequestContext| { + let flag = DropFlag(Arc::clone(&handler_dropped)); + async move { + // An origin body that never ends, 64 KiB at a time. + let chunk = Bytes::from(vec![1u8; 64 * 1024]); + let stream = futures::stream::unfold(flag, move |flag| { + let chunk = chunk.clone(); + async move { + tokio::task::yield_now().await; + Some((Ok::<_, std::io::Error>(chunk), flag)) + } + }); + Ok::<_, EdgeError>(EdgeBody::from_stream(stream)) + } + }) + .build(); + let base = start(router).await; + + let mut response = reqwest::get(format!("{base}/media")) + .await + .expect("the head should arrive"); + response + .chunk() + .await + .expect("the first chunk should be readable") + .expect("the first chunk should arrive"); + drop(response); + + tokio::time::timeout(Duration::from_secs(5), async { + while !dropped.load(std::sync::atomic::Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the body, and the origin read it holds, should be dropped once the client leaves"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_streamed_body_reaches_the_client_before_it_finishes() { + let release = Arc::new(tokio::sync::Notify::new()); + let handler_release = Arc::clone(&release); + let router = RouterService::builder() + .get("/media", move |_ctx: RequestContext| { + let release = Arc::clone(&handler_release); + async move { + let response = edgezero_core::http::response_builder() + .status(200) + .header("x-origin", "kept") + .header(header::CONTENT_TYPE, "video/mp4") + .body(gated_body(b"first-", release, Ok(b"second"))) + .expect("should build the streamed response"); + Ok::<_, EdgeError>(response) + } + }) + .build(); + let base = start(router).await; + + let mut response = tokio::time::timeout( + Duration::from_secs(5), + reqwest::get(format!("{base}/media")), + ) + .await + .expect("the head should arrive while the body is still open") + .expect("the request should succeed"); + assert_eq!(response.status().as_u16(), 200); + assert_eq!( + response + .headers() + .get("x-origin") + .and_then(|value| value.to_str().ok()), + Some("kept"), + "should deliver the router's headers" + ); + let first = tokio::time::timeout(Duration::from_secs(5), response.chunk()) + .await + .expect("the first chunk should arrive before the body finishes") + .expect("the first chunk should be readable") + .expect("the body should not end after the first chunk"); + assert_eq!(first.as_ref(), b"first-"); + + release.notify_one(); + let mut rest = Vec::new(); + while let Some(chunk) = tokio::time::timeout(Duration::from_secs(5), response.chunk()) + .await + .expect("the rest of the body should arrive once released") + .expect("the rest of the body should be readable") + { + rest.extend_from_slice(&chunk); + } + assert_eq!(rest, b"second", "should deliver the released chunk and end"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_body_that_fails_mid_stream_is_not_delivered_as_complete() { + let release = Arc::new(tokio::sync::Notify::new()); + let handler_release = Arc::clone(&release); + let router = RouterService::builder() + .get("/media", move |_ctx: RequestContext| { + let release = Arc::clone(&handler_release); + async move { + Ok::<_, EdgeError>(gated_body( + b"partial", + release, + Err("origin connection reset"), + )) + } + }) + .build(); + let base = start(router).await; + + let mut response = reqwest::get(format!("{base}/media")) + .await + .expect("the head should arrive"); + let first = response + .chunk() + .await + .expect("the first chunk should be readable") + .expect("the first chunk should arrive"); + assert_eq!(first.as_ref(), b"partial"); + + release.notify_one(); + let outcome = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match response.chunk().await { + Ok(Some(_)) => {} + Ok(None) => return Ok(()), + Err(error) => return Err(error), + } + } + }) + .await + .expect("the failure should surface promptly"); + assert!( + outcome.is_err(), + "a failed stream must abort the response rather than end it cleanly" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_buffered_body_and_an_unknown_route_answer_as_before() { + async fn hello(_ctx: RequestContext) -> Result<&'static str, EdgeError> { + Ok("hello") + } + let router = RouterService::builder().get("/hello", hello).build(); + let base = start(router).await; + + let hello = reqwest::get(format!("{base}/hello")) + .await + .expect("the buffered route should answer"); + assert_eq!(hello.status().as_u16(), 200); + assert_eq!( + hello.text().await.expect("the body should be readable"), + "hello" + ); + + let missing = reqwest::get(format!("{base}/missing")) + .await + .expect("an unknown route should answer"); + assert_eq!( + missing.status().as_u16(), + 404, + "the router's own not-found answer should pass through" + ); + } +} diff --git a/crates/trusted-server-adapter-axum/tests/shipped_modules.rs b/crates/trusted-server-adapter-axum/tests/shipped_modules.rs index a72162abb8..71992faad0 100644 --- a/crates/trusted-server-adapter-axum/tests/shipped_modules.rs +++ b/crates/trusted-server-adapter-axum/tests/shipped_modules.rs @@ -1,23 +1,13 @@ //! What the module builders the Axum adapter links give a deployment, proven //! from outside the adapter: the stock loading cache, each module's own //! deploy rules and secret settings, the diagnostics module's request -//! preparer, the middleware the template and the guide document, the ad -//! server's decision between planned bids, and the maintainers every crate -//! outside core names. +//! preparer, the ad server's decision between planned bids, and the maintainers +//! every crate outside core names. #![allow(clippy::panic, reason = "tests use panic-on-failure helpers")] use trusted_server_adapter_axum::app::shipped_module_builders; -use trusted_server_core::integrations::{ - IntegrationRegistry, SuppliedMiddleware, supplied_middleware, -}; -use trusted_server_core::middleware::{HTML_MEDIA_TYPE, MiddlewarePhase}; - -/// The middleware the linked modules and core supply, with the integration -/// that supplies each, in the order the registry registers them. -fn listed_middleware() -> Vec<(&'static str, SuppliedMiddleware)> { - supplied_middleware(&shipped_module_builders()) -} +use trusted_server_core::integrations::IntegrationRegistry; /// A feature selects the stock loading cache by the module's short name, so /// a stock build keeps what it loads rather than asking each source every @@ -309,106 +299,6 @@ fn the_linked_list_attaches_the_diagnostics_module_s_request_preparer() { ); } -/// The template's entries name every middleware the linked modules and core -/// supply, in the order the registry registers them, which is the order a -/// deployment that runs them all names them in. -#[test] -fn the_template_s_entries_name_the_listed_middleware() { - use trusted_server_core::settings::Settings; - use trusted_server_core::test_support::template::{ - template_with_resolved_required_secrets, uncomment_block, - }; - - let template = uncomment_block( - &uncomment_block(&template_with_resolved_required_secrets(), "[[fetch]]"), - "[[serve]]", - ); - let settings = - Settings::from_toml(&template).expect("should load the template with its entries"); - let listed = listed_middleware(); - - for phase in MiddlewarePhase::ALL { - let entries = settings.phase_entries(phase).entries(); - let mut media_types: Vec<&str> = Vec::new(); - for (_, row) in listed.iter().filter(|(_, row)| row.phase == phase) { - if !media_types.contains(&row.media_type) { - media_types.push(row.media_type); - } - } - for media_type in media_types { - let named_in_list: Vec<&str> = listed - .iter() - .filter(|(_, row)| row.phase == phase && row.media_type == media_type) - .map(|(_, row)| row.name) - .collect(); - let covering: Vec<_> = entries - .iter() - .filter(|entry| entry.media_type == media_type) - .collect(); - if media_type == HTML_MEDIA_TYPE || covering.iter().any(|entry| entry.path.is_none()) { - // One entry covers every page, for HTML and for a media - // type a page's own responses come in. - assert_eq!( - covering.len(), - 1, - "should document one [[{phase}]] entry for {media_type}" - ); - assert!( - covering[0].path.is_none(), - "should document the [[{phase}]] entry that covers every page" - ); - } else { - // A file's middleware is placed on the file's own path, one - // entry for each file. - assert!( - !covering.is_empty(), - "should document a [[{phase}]] entry on a path for each {media_type} file" - ); - } - let named: Vec<&str> = covering - .iter() - .flat_map(|entry| entry.middleware.iter().map(String::as_str)) - .collect(); - assert_eq!( - named, named_in_list, - "should document every {phase} middleware the linked modules supply for \ - {media_type}, in the listed order" - ); - } - } -} - -/// The guide's table of middleware lists what the linked modules and core -/// supply, each with its phase, in the order the registry registers them. -#[test] -fn the_guide_s_table_lists_the_same_middleware() { - const GUIDE: &str = include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../docs/guide/configuration.md" - )); - - // A row of the table is a name and then the list it is written in. - let documented: Vec<(String, String)> = GUIDE - .lines() - .filter_map(|line| { - let mut cells = line.split('|').map(str::trim).skip(1); - let name = cells.next()?.strip_prefix('`')?.strip_suffix('`')?; - let list = cells.next()?.strip_prefix("`[[")?.strip_suffix("]]`")?; - Some((name.to_owned(), list.to_owned())) - }) - .collect(); - let listed: Vec<(String, String)> = listed_middleware() - .iter() - .map(|(_, row)| (row.name.to_owned(), row.phase.to_string())) - .collect(); - - assert_eq!( - documented, listed, - "should document the middleware the linked modules supply, each with its phase, in \ - the listed order" - ); -} - /// The ad server decides between the bids APS returned, and the winner /// keeps the seat the exchange returned, APS's own bidder name and its /// renderer. diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index 1d98405d7d..3498a2fdc8 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -69,6 +69,7 @@ js-sys = { workspace = true } worker = { workspace = true } [dev-dependencies] +anyhow = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } base64 = { workspace = true } edgezero-core = { workspace = true } diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index b73760c7af..bc9d22ae3d 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -530,7 +530,9 @@ impl AppState { if let Some(module) = self.registry.device_module() { services = services.with_device_module(module); } - services = services.with_identifier_modules(self.registry.identifier_modules()); + services = services + .with_identifier_modules(self.registry.identifier_modules()) + .with_media_signer(self.registry.media_signer()); // The publisher this request resolved to names every key in a shared // store and every name the browser holds. let prefixes = self.registry.prefixes( @@ -1103,8 +1105,15 @@ async fn dispatch_asset_fallback( let asset_route = &state.settings.proxy.asset_routes[index]; let answer = match state.registry.route_source(&state.settings, index) { Ok(source) => { - handle_asset_proxy_request(&state.settings, services, req, asset_route, source.as_ref()) - .await + handle_asset_proxy_request( + &state.settings, + services, + &state.registry, + req, + asset_route, + source.as_ref(), + ) + .await } Err(report) => Err(report), }; diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 8bb37cd902..7885edef65 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -23,6 +23,9 @@ use trusted_server_core::platform::{ PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, }; +#[cfg(any(target_arch = "wasm32", test))] +mod response_stream; + // --------------------------------------------------------------------------- // Noop stubs — used when a handle is absent (native CI, missing binding) // --------------------------------------------------------------------------- @@ -215,6 +218,13 @@ struct CloudflarePendingResponse { /// [`worker::Fetch`]-backed HTTP client for the Cloudflare Workers runtime. /// +/// [`send`](PlatformHttpClient::send) preserves the origin body as a +/// `Body::Stream` when the request asks for it, reading the Workers +/// `ReadableStream` one runtime chunk at a time, see +/// [`response_stream::bounded_origin_stream`]. A streamed body honors the +/// caller's exact `max_response_body_bytes`, if any, instead of the buffered +/// path's 10 MiB ceiling. +/// /// # Multi-provider auction limitation /// /// `send_async` eagerly awaits each request before returning, so @@ -223,6 +233,8 @@ struct CloudflarePendingResponse { /// request launches. `select` keeps a defense-in-depth rejection for more /// than one pending request. Configure a single auction provider for /// Cloudflare Workers, or use the Fastly adapter for parallel DSP fan-out. +/// That path stays buffered, so it refuses a request that asks for a +/// streamed response. /// /// Per-provider timeouts baked into the backend name are not enforced at the /// fetch layer; the Workers runtime's global CPU budget (~30 s on paid plans) @@ -281,6 +293,17 @@ fn response_carries_body(request_is_head: bool, status: u16) -> bool { !request_is_head && !(100..200).contains(&status) && !matches!(status, 204 | 205 | 304) } +/// Decoded bodies have no usable wire length; HEAD and 304 keep the +/// selected representation's length rather than advertising an empty asset. +#[cfg(any(target_arch = "wasm32", test))] +fn streamed_content_length(status: u16, unencoded: bool, claimed: Option) -> Option { + if unencoded && !(100..200).contains(&status) && !matches!(status, 204 | 205) { + claimed + } else { + None + } +} + /// Collect the lowercased tokens listed in the upstream `Connection:` header(s). /// /// Every header named here is hop-by-hop (RFC 7230 §6.1) and must not be @@ -371,26 +394,27 @@ impl CloudflareHttpClient { ) -> Result> { use worker::{CacheMode, Fetch, Headers, Method, Request}; - // The Cloudflare fetch path cannot honor Image Optimizer metadata, and - // it always buffers the response body (see below). The - // `PlatformHttpRequest` contract requires adapters that cannot honor - // these to reject rather than silently drop the behavior, so surface a - // typed error instead of returning an untransformed or fully-buffered - // response. Only an asset route sets these fields. A route that - // enables the Image Optimizer answers with this error, and none asks - // for a stream, because this client reports that it cannot stream. + // The Cloudflare fetch path cannot honor Image Optimizer metadata. + // The `PlatformHttpRequest` contract requires adapters that cannot + // honor it to reject rather than silently drop the behavior, so + // surface a typed error instead of returning an untransformed + // response. Only an asset route sets it. if request.image_optimizer.is_some() { return Err(Report::new(PlatformError::HttpClient) .attach("Image Optimizer is not supported on the Cloudflare Workers runtime")); } - if request.stream_response { - return Err(Report::new(PlatformError::HttpClient).attach( - "streaming response bodies are not supported on the Cloudflare Workers runtime", - )); - } let cache_mode = outbound_cache_mode(&request.cache_intent); - let response_limit = effective_response_body_limit(request.max_response_body_bytes); + let stream_response = request.stream_response; + // A streamed body holds one chunk at a time, so only the caller's own + // limit applies to it. A buffered body is also held to the ceiling. + let response_limit = if stream_response { + request.max_response_body_bytes + } else { + Some(effective_response_body_limit( + request.max_response_body_bytes, + )) + }; let request_is_head = request.request.method() == edgezero_core::http::Method::HEAD; let uri = request.request.uri().to_string(); @@ -456,23 +480,36 @@ impl CloudflareHttpClient { let status = resp.status_code(); - // Content-Length permits an early refusal, but the stream collector is + // Content-Length permits an early refusal, but the body reader is // authoritative for missing or dishonest lengths. let response_body_expected = response_carries_body(request_is_head, status); + let claimed_len = resp + .headers() + .get("content-length") + .ok() + .flatten() + .and_then(|v| v.trim().parse::().ok()); if response_body_expected - && let Some(claimed_len) = resp - .headers() - .get("content-length") - .ok() - .flatten() - .and_then(|v| v.trim().parse::().ok()) - && claimed_len > response_limit + && let Some(limit) = response_limit + && let Some(claimed_len) = claimed_len + && u64::try_from(limit).is_ok_and(|limit| claimed_len > limit) { return Err(Report::new(PlatformError::HttpClient).attach(format!( "origin Content-Length {claimed_len} exceeds \ - {response_limit}-byte response body limit" + {limit}-byte response body limit" ))); } + // The runtime decodes a content encoding, after which the declared + // length describes bytes the stream never yields. + let body_is_unencoded = resp + .headers() + .get("content-encoding") + .ok() + .flatten() + .is_none_or(|encoding| { + let encoding = encoding.trim(); + encoding.is_empty() || encoding.eq_ignore_ascii_case("identity") + }); let connection_tokens = response_connection_tokens(&resp); let mut edge_builder = edgezero_core::http::response_builder().status(status); @@ -498,18 +535,29 @@ impl CloudflareHttpClient { } edge_builder = edge_builder.header(name.as_str(), value.as_bytes()); } - let body_bytes = if response_body_expected { + if (stream_response || !response_body_expected) + && let Some(length) = streamed_content_length(status, body_is_unencoded, claimed_len) + { + edge_builder = edge_builder.header(edgezero_core::http::header::CONTENT_LENGTH, length); + } + let body = if !response_body_expected { + edgezero_core::body::Body::empty() + } else if stream_response { let stream = resp.stream().change_context(PlatformError::HttpClient)?; - collect_bounded_response_stream(stream, response_limit).await? + let expected_length = claimed_len.filter(|_| body_is_unencoded); + response_stream::bounded_origin_stream(stream, response_limit, expected_length) } else { - Vec::new() + let limit = response_limit.unwrap_or(MAX_PLATFORM_RESPONSE_BODY_BYTES); + let stream = resp.stream().change_context(PlatformError::HttpClient)?; + let body_bytes = collect_bounded_response_stream(stream, limit).await?; + edge_builder = edge_builder.header( + edgezero_core::http::header::CONTENT_LENGTH, + body_bytes.len(), + ); + edgezero_core::body::Body::from(body_bytes) }; - edge_builder = edge_builder.header( - edgezero_core::http::header::CONTENT_LENGTH, - body_bytes.len(), - ); let edge_resp = edge_builder - .body(edgezero_core::body::Body::from(body_bytes)) + .body(body) .change_context(PlatformError::HttpClient)?; Ok(PlatformResponse::new(edge_resp).with_backend_name(request.backend_name)) @@ -533,10 +581,15 @@ impl PlatformHttpClient for CloudflareHttpClient { false } + fn supports_streaming_responses(&self) -> bool { + true + } + async fn send_async( &self, request: PlatformHttpRequest, ) -> Result> { + reject_buffered_stream_request(request.stream_response)?; let backend_name = request.backend_name.clone(); let response = self.execute(request).await?; @@ -895,11 +948,41 @@ fn reject_multi_provider_fanout(len: usize) -> Result<(), Report> Ok(()) } +/// Refuse a streamed response on the buffered `send_async` path. +/// +/// `send_async` stores a completed, buffered response for `select`, so a +/// request that asks for a stream must go through `send` instead of being +/// silently buffered. The check runs before the fetch, so a refused request +/// never reaches the origin. +#[cfg(any(target_arch = "wasm32", test))] +fn reject_buffered_stream_request(stream_response: bool) -> Result<(), Report> { + if stream_response { + return Err(Report::new(PlatformError::HttpClient).attach( + "CloudflareHttpClient streams responses through send alone; send_async buffers", + )); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; use edgezero_core::context::RequestContext; + #[test] + fn streamed_lengths_describe_the_identity_representation() { + let large = 64 * 1024 * 1024; + assert_eq!(streamed_content_length(200, true, Some(large)), Some(large)); + assert_eq!(streamed_content_length(206, true, Some(1024)), Some(1024)); + assert_eq!(streamed_content_length(304, true, Some(large)), Some(large)); + assert_eq!(streamed_content_length(200, false, Some(1024)), None); + assert_eq!(streamed_content_length(304, false, Some(1024)), None); + assert_eq!(streamed_content_length(200, true, None), None); + for status in [101, 204, 205] { + assert_eq!(streamed_content_length(status, true, Some(large)), None); + } + } + #[cfg(target_arch = "wasm32")] #[test] fn outbound_request_creation_sets_manual_redirect_mode() { @@ -1206,6 +1289,18 @@ mod tests { ); } + #[test] + fn send_async_refuses_a_streamed_response_and_admits_a_buffered_one() { + assert!( + reject_buffered_stream_request(true).is_err(), + "the buffered send_async path must refuse a streamed response" + ); + assert!( + reject_buffered_stream_request(false).is_ok(), + "a buffered request should still use send_async" + ); + } + #[test] fn reject_multi_provider_fanout_rejects_five_providers() { let err = reject_multi_provider_fanout(5).expect_err("should reject five providers"); diff --git a/crates/trusted-server-adapter-cloudflare/src/platform/response_stream.rs b/crates/trusted-server-adapter-cloudflare/src/platform/response_stream.rs new file mode 100644 index 0000000000..904fff4729 --- /dev/null +++ b/crates/trusted-server-adapter-cloudflare/src/platform/response_stream.rs @@ -0,0 +1,307 @@ +//! Origin response bodies passed on as the Workers runtime delivers them. +//! +//! A streamed response never holds more than the chunk the runtime handed +//! over last. The caller's byte limit and the origin's declared length are +//! checked chunk by chunk, so an oversized, truncated or failed origin body +//! ends the stream with an error instead of reaching the caller as a +//! complete one. + +use std::fmt::Display; +use std::pin::Pin; + +use bytes::Bytes; +use futures::{Stream, StreamExt as _}; + +/// The source and the number of bytes already yielded from it. +struct Progress { + source: Pin>, + yielded: usize, +} + +/// A byte count in the unit of a declared `Content-Length`. A count no +/// `u64` can hold is past any declared length. +fn declared_units(bytes: usize) -> u64 { + u64::try_from(bytes).unwrap_or(u64::MAX) +} + +/// Wrap a runtime body stream as a [`Body::Stream`](edgezero_core::body::Body::Stream). +/// +/// `limit` is the caller's `max_response_body_bytes`. The chunk that would +/// carry the running total past it is replaced by an error, and nothing +/// after it is read. +/// +/// `expected_length` is the origin's `Content-Length` when the runtime hands +/// the body over unchanged. The Workers `ByteStream` reports a JavaScript +/// `aborted` stream as a clean end, so an origin connection cut short would +/// otherwise look like a complete body. A body that ends short of the +/// declared length, or runs past it, therefore ends with an error. Pass +/// `None` when the runtime decoded a content encoding, because the declared +/// length then describes the encoded bytes. +/// +/// A source error is passed on as a stream error and ends the stream. +pub(super) fn bounded_origin_stream( + source: S, + limit: Option, + expected_length: Option, +) -> edgezero_core::body::Body +where + S: Stream> + 'static, + B: Into, + E: Display, +{ + let initial = Progress { + source: Box::pin(source), + yielded: 0, + }; + let stream = futures::stream::unfold(Some(initial), move |state| async move { + let mut progress = state?; + match progress.source.next().await { + None => { + if let Some(expected) = expected_length + && declared_units(progress.yielded) != expected + { + return Some(( + Err(std::io::Error::other(format!( + "Workers response body ended after {} of its declared {expected} bytes", + progress.yielded + ))), + None, + )); + } + None + } + Some(Err(error)) => Some(( + Err(std::io::Error::other(format!( + "failed to read Workers response body: {error}" + ))), + None, + )), + Some(Ok(chunk)) => { + let chunk: Bytes = chunk.into(); + let Some(next_yielded) = progress.yielded.checked_add(chunk.len()) else { + return Some(( + Err(std::io::Error::other( + "Workers response body byte count overflowed", + )), + None, + )); + }; + if let Some(limit) = limit + && next_yielded > limit + { + return Some(( + Err(std::io::Error::other(format!( + "Workers response body exceeded caller limit of {limit} bytes" + ))), + None, + )); + } + if let Some(expected) = expected_length + && declared_units(next_yielded) > expected + { + return Some(( + Err(std::io::Error::other(format!( + "Workers response body ran past its declared {expected} bytes" + ))), + None, + )); + } + progress.yielded = next_yielded; + Some((Ok(chunk), Some(progress))) + } + } + }); + edgezero_core::body::Body::from_stream(stream) +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::FutureExt as _; + + type ChunkResult = Result, std::io::Error>; + + fn into_stream( + body: edgezero_core::body::Body, + ) -> futures::stream::LocalBoxStream<'static, Result> { + body.into_stream() + .expect("a streamed origin body should stay a stream") + } + + #[tokio::test] + async fn each_chunk_is_yielded_before_the_origin_finishes() { + let (sender, receiver) = futures::channel::mpsc::unbounded::(); + sender + .unbounded_send(Ok(b"ab".to_vec())) + .expect("should queue the first chunk"); + let mut stream = into_stream(bounded_origin_stream(receiver, None, None)); + + let first = stream + .next() + .await + .expect("the first chunk should arrive") + .expect("the first chunk should be readable"); + assert_eq!( + first.as_ref(), + b"ab", + "should pass the first chunk on as it arrived" + ); + assert!( + stream.next().now_or_never().is_none(), + "should wait for the origin rather than report an end it has not sent" + ); + + sender + .unbounded_send(Ok(b"cd".to_vec())) + .expect("should queue the second chunk"); + drop(sender); + let second = stream + .next() + .await + .expect("the second chunk should arrive") + .expect("the second chunk should be readable"); + assert_eq!(second.as_ref(), b"cd", "should pass the later chunk on"); + assert!(stream.next().await.is_none(), "should end with the origin"); + } + + #[tokio::test] + async fn an_unbounded_stream_passes_more_than_the_buffered_ceiling() { + // Eleven 1 MiB chunks cross the 10 MiB ceiling the buffered path + // enforces. The chunks share one allocation, so the test holds 1 MiB. + let chunk = Bytes::from(vec![7u8; 1024 * 1024]); + let chunks = std::iter::repeat_n(chunk, 11).map(Ok::<_, std::io::Error>); + let mut stream = into_stream(bounded_origin_stream( + futures::stream::iter(chunks), + None, + None, + )); + + let mut total = 0usize; + while let Some(chunk) = stream.next().await { + total += chunk.expect("an unbounded chunk should be readable").len(); + } + assert_eq!(total, 11 * 1024 * 1024, "should pass every byte on"); + } + + #[tokio::test] + async fn the_chunk_that_crosses_the_caller_limit_ends_the_stream() { + let source = futures::stream::iter([ + Ok::<_, std::io::Error>(b"ab".to_vec()), + Ok(b"cd".to_vec()), + Ok(b"e".to_vec()), + Ok(b"never read".to_vec()), + ]); + let mut stream = into_stream(bounded_origin_stream(source, Some(4), None)); + + for expected in [b"ab", b"cd"] { + let chunk = stream + .next() + .await + .expect("a chunk within the limit should arrive") + .expect("a chunk within the limit should be readable"); + assert_eq!( + chunk.as_ref(), + expected, + "should pass chunks up to the exact limit" + ); + } + let error = stream + .next() + .await + .expect("the crossing chunk should surface") + .expect_err("the crossing chunk must be refused"); + assert!( + error.to_string().contains("limit of 4 bytes"), + "should report the enforced limit: {error}" + ); + assert!( + stream.next().await.is_none(), + "should read nothing after the refused chunk" + ); + } + + #[tokio::test] + async fn an_origin_failure_is_passed_on_and_ends_the_stream() { + let source = futures::stream::iter([ + Ok(b"ab".to_vec()), + Err(std::io::Error::other("connection reset by origin")), + Ok(b"never read".to_vec()), + ]); + let mut stream = into_stream(bounded_origin_stream(source, Some(64), None)); + + let first = stream + .next() + .await + .expect("the chunk before the failure should arrive") + .expect("the chunk before the failure should be readable"); + assert_eq!(first.as_ref(), b"ab"); + let error = stream + .next() + .await + .expect("the failure should surface") + .expect_err("the origin failure must not look like data"); + assert!( + error.to_string().contains("connection reset by origin"), + "should carry the origin's failure: {error}" + ); + assert!(stream.next().await.is_none(), "should end at the failure"); + } + + #[tokio::test] + async fn a_body_cut_short_of_its_declared_length_is_an_error() { + // The Workers stream reports an aborted origin as a clean end. + let source = futures::stream::iter([Ok::<_, std::io::Error>(b"ab".to_vec())]); + let mut stream = into_stream(bounded_origin_stream(source, None, Some(4))); + + let first = stream + .next() + .await + .expect("the delivered bytes should arrive") + .expect("the delivered bytes should be readable"); + assert_eq!(first.as_ref(), b"ab"); + let error = stream + .next() + .await + .expect("the early end should surface") + .expect_err("a truncated body must not end cleanly"); + assert!( + error.to_string().contains("2 of its declared 4 bytes"), + "should report how much of the declared body arrived: {error}" + ); + } + + #[tokio::test] + async fn a_body_running_past_its_declared_length_is_an_error() { + let source = + futures::stream::iter([Ok::<_, std::io::Error>(b"abc".to_vec()), Ok(b"de".to_vec())]); + let mut stream = into_stream(bounded_origin_stream(source, None, Some(4))); + + stream + .next() + .await + .expect("the bytes within the declared length should arrive") + .expect("the bytes within the declared length should be readable"); + let error = stream + .next() + .await + .expect("the overrun should surface") + .expect_err("bytes past the declared length must be refused"); + assert!( + error.to_string().contains("declared 4 bytes"), + "should report the declared length: {error}" + ); + } + + #[tokio::test] + async fn a_body_matching_its_declared_length_ends_cleanly() { + let source = + futures::stream::iter([Ok::<_, std::io::Error>(b"ab".to_vec()), Ok(b"cd".to_vec())]); + let mut stream = into_stream(bounded_origin_stream(source, Some(4), Some(4))); + + let mut body = Vec::new(); + while let Some(chunk) = stream.next().await { + body.extend_from_slice(&chunk.expect("a declared-length body should be readable")); + } + assert_eq!(body, b"abcd", "should deliver the whole declared body"); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index db6fa5815d..709eef8204 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -614,7 +614,9 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime if let Some(module) = state.registry.device_module() { services = services.with_device_module(module); } - let services = services.with_identifier_modules(state.registry.identifier_modules()); + let services = services + .with_identifier_modules(state.registry.identifier_modules()) + .with_media_signer(state.registry.media_signer()); // The publisher this request resolved to, its tenant included, names // every key in a shared store and every name the browser holds. let prefixes = state.registry.prefixes( @@ -1382,8 +1384,15 @@ async fn dispatch_asset_fallback( let asset_route = &state.settings.proxy.asset_routes[index]; let answer = match state.registry.route_source(&state.settings, index) { Ok(source) => { - handle_asset_proxy_request(&state.settings, services, req, asset_route, source.as_ref()) - .await + handle_asset_proxy_request( + &state.settings, + services, + &state.registry, + req, + asset_route, + source.as_ref(), + ) + .await } Err(report) => Err(report), }; diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 21fa640f2c..6a5c169886 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -296,6 +296,16 @@ fn apply_fastly_image_optimizer_params( Ok(()) } +/// The Image Optimizer query parameter that names the metadata it keeps. +/// Its `c2pa` filter keeps the input's C2PA manifest and adds the +/// transformations the optimizer performed, and `copyright,c2pa` applies +/// both filters. The SDK's `Metadata` options offer only `copyright`, so +/// the filters are sent as the query parameter the optimizer reads beside +/// its options. The optimizer removes the parameters it knows from its +/// origin request, so an origin signature over the address without it +/// still holds. +const IMAGE_OPTIMIZER_METADATA_PARAM: &str = "metadata"; + fn apply_fastly_image_optimizer( req: &mut fastly::Request, options: PlatformImageOptimizerOptions, @@ -305,6 +315,35 @@ fn apply_fastly_image_optimizer( fastly_options.preserve_query_string_on_origin_request = Some(options.preserve_query_string_on_origin_request); apply_fastly_image_optimizer_params(&mut fastly_options, options.params)?; + if options.preserve_c2pa { + // An option outranks a query parameter of the same name, so a + // copyright filter set as an option moves into the query beside + // `c2pa`, as does one the address already asks for. Every other + // parameter keeps its exact bytes. + let mut copyright = fastly_options.metadata.take().is_some(); + let mut url = req.get_url_mut(); + let mut pairs: Vec = Vec::new(); + for pair in url + .query() + .unwrap_or_default() + .split('&') + .filter(|pair| !pair.is_empty()) + { + let (key, value) = pair.split_once('=').unwrap_or((pair, "")); + if key == IMAGE_OPTIMIZER_METADATA_PARAM { + copyright |= value + .to_ascii_lowercase() + .replace("%2c", ",") + .split(',') + .any(|filter| filter.trim() == "copyright"); + } else { + pairs.push(pair.to_owned()); + } + } + let filters = if copyright { "copyright,c2pa" } else { "c2pa" }; + pairs.push(format!("{IMAGE_OPTIMIZER_METADATA_PARAM}={filters}")); + url.set_query(Some(&pairs.join("&"))); + } req.set_image_optimizer(fastly_options); Ok(()) } @@ -1594,6 +1633,53 @@ mod tests { ); } + #[test] + fn image_optimizer_c2pa_preservation_is_asked_only_when_requested() { + let options = || { + PlatformImageOptimizerOptions::new( + "us_east", + PlatformImageOptimizerParams { + width: Some(320), + ..PlatformImageOptimizerParams::default() + }, + ) + }; + + let mut plain = fastly::Request::get("https://assets.example.com/photo.jpg"); + apply_fastly_image_optimizer(&mut plain, options()).expect("should translate the options"); + assert_eq!( + plain.get_query_str(), + None, + "an unsigned route should keep the optimizer's default metadata removal" + ); + + let mut preserved = fastly::Request::get("https://assets.example.com/photo.jpg?v=2"); + apply_fastly_image_optimizer(&mut preserved, options().with_c2pa_preserved()) + .expect("should translate the options"); + assert_eq!( + preserved.get_query_str(), + Some("v=2&metadata=c2pa"), + "should ask for C2PA preservation and keep every other parameter as sent" + ); + + for asked in [ + "metadata=copyright", + "metadata=Copyright%2Cc2pa", + "metadata=c2pa,copyright", + ] { + let mut with_copyright = fastly::Request::get(format!( + "https://assets.example.com/photo.jpg?v=2&{asked}&x-id=a%20b" + )); + apply_fastly_image_optimizer(&mut with_copyright, options().with_c2pa_preserved()) + .expect("should translate the options"); + assert_eq!( + with_copyright.get_query_str(), + Some("v=2&x-id=a%20b&metadata=copyright,c2pa"), + "{asked}: should apply both filters once, keeping the copyright filter" + ); + } + } + #[test] fn fastly_platform_http_client_supports_pending_streaming_responses() { let client = FastlyPlatformHttpClient; diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index 333998b51e..1b000cacd3 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -29,6 +29,8 @@ edgezero-core = { workspace = true } error-stack = { workspace = true } flate2 = { workspace = true } futures = { workspace = true } +# Frames for the streamed response body handed to the Spin host (src/lib.rs). +http-body = { workspace = true } http-body-util = { workspace = true } log = { workspace = true } trusted-server-ad-server-iabtl = { workspace = true } diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index abb60761c2..2b64aaa968 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -481,7 +481,9 @@ impl AppState { if let Some(module) = self.registry.device_module() { services = services.with_device_module(module); } - services = services.with_identifier_modules(self.registry.identifier_modules()); + services = services + .with_identifier_modules(self.registry.identifier_modules()) + .with_media_signer(self.registry.media_signer()); // The publisher this request resolved to names every key in a shared // store and every name the browser holds. let prefixes = self.registry.prefixes( @@ -1265,8 +1267,15 @@ async fn dispatch_asset_fallback( let asset_route = &state.settings.proxy.asset_routes[index]; let answer = match state.registry.route_source(&state.settings, index) { Ok(source) => { - handle_asset_proxy_request(&state.settings, services, req, asset_route, source.as_ref()) - .await + handle_asset_proxy_request( + &state.settings, + services, + &state.registry, + req, + asset_route, + source.as_ref(), + ) + .await } Err(report) => Err(report), }; diff --git a/crates/trusted-server-adapter-spin/src/lib.rs b/crates/trusted-server-adapter-spin/src/lib.rs index 502940a017..937e35862e 100644 --- a/crates/trusted-server-adapter-spin/src/lib.rs +++ b/crates/trusted-server-adapter-spin/src/lib.rs @@ -5,15 +5,222 @@ pub mod logging; pub mod middleware; pub mod platform; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_adapter_spin::key_value_store::{DEFAULT_MAX_LIST_KEYS, SpinKvStore}; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_core::app::{Hooks, StoreMetadata}; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_core::env_config::EnvConfig; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_core::store_registry::KvRegistry; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use spin_sdk::http::{IntoResponse, Request}; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use spin_sdk::http_service; +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +use futures::StreamExt as _; + +/// The body of every response this component sends: a buffered body as the +/// router built it, or a streamed body passed to the host frame by frame. +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +type ResponseBody = http_body_util::Either< + http_body_util::Full, + http_body_util::StreamBody< + futures::stream::LocalBoxStream< + 'static, + Result, anyhow::Error>, + >, + >, +>; + #[cfg(all(feature = "spin", target_arch = "wasm32"))] #[http_service] -// FORCED: edgezero_adapter_spin::run_app returns anyhow::Result — EdgeZero SDK constraint, not a project choice. +// FORCED: the Spin `http_service` boundary takes this `anyhow::Result`, the +// error type `edgezero_adapter_spin::run_app` returned before this streaming +// dispatch replaced it, so failures reach the host exactly as they did. async fn handle(req: Request) -> anyhow::Result { logging::init_logger(); - edgezero_adapter_spin::run_app::(req).await + dispatch::(req).await +} + +/// Dispatch a request through the router and hand any streamed body to the +/// host as it is produced. +/// +/// This replaces `edgezero_adapter_spin::run_app`, whose response conversion +/// collects a `Body::Stream` into one buffer, capped at 16 MiB, before the +/// host sees a byte. `EdgeZero` keeps the registry construction that wrapper +/// performs private, so the declared KV stores are opened here with the +/// same settings and error, see [`kv_registry`]. Request conversion and +/// routing are `EdgeZero`'s own. +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +async fn dispatch( + req: Request, +) -> anyhow::Result> { + let app = A::build_app(); + let stores = A::stores(); + // `run_app` would also open declared config and secret stores. This app + // declares neither, so one appearing here must fail loudly rather than + // reach the router unbound. + if stores.config.is_some() || stores.secrets.is_some() { + anyhow::bail!( + "the streaming Spin dispatch binds declared KV stores alone, \ + but the app declares a config or secret store" + ); + } + let env = EnvConfig::from_env(); + let kv_registry = kv_registry(stores.kv, &env).await?; + let mut core_request = edgezero_adapter_spin::request::into_core_request(req).await?; + if let Some(registry) = kv_registry { + core_request.extensions_mut().insert(registry); + } + let response = app.router().oneshot(core_request).await?; + let (parts, body) = response.into_parts(); + Ok(spin_sdk::http::Response::::from_parts( + parts, + response_body(body), + )) +} + +/// Open every declared KV store, as `EdgeZero`'s Spin dispatch does. +/// +/// Each id opens the label `EDGEZERO__STORES__KV____NAME` names, with +/// the list cap `EDGEZERO__STORES__KV____MAX_LIST_KEYS` sets. A declared +/// store that cannot be opened fails the request, because a declared store +/// is required. +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +async fn kv_registry( + meta: Option, + env: &EnvConfig, +) -> anyhow::Result> { + let Some(meta) = meta else { + return Ok(None); + }; + let mut by_id = std::collections::BTreeMap::new(); + for id in meta.ids { + let label = env.store_name("kv", id); + let max_list_keys = env + .store_setting("kv", id, "MAX_LIST_KEYS") + .and_then(|raw| raw.parse::().ok()) + .unwrap_or(DEFAULT_MAX_LIST_KEYS); + let store = SpinKvStore::open_with_max_list_keys(&label, max_list_keys) + .await + .map_err(|err| { + anyhow::anyhow!( + "Spin KV store '{label}' (id `{id}`) is explicitly configured but could not be opened: {err}" + ) + })?; + by_id.insert( + (*id).to_owned(), + edgezero_core::key_value_store::KvHandle::new(std::sync::Arc::new(store)), + ); + } + Ok(KvRegistry::from_parts(by_id, meta.default.to_owned())) +} + +/// The host body for a router body. +/// +/// A buffered body stays one buffer. A streamed body becomes one frame per +/// chunk, read only when the host asks for the next frame, so it never +/// holds more than one chunk. A chunk error is passed on as a body error, +/// which aborts the response instead of ending it as though it were +/// complete. +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +fn response_body(body: edgezero_core::body::Body) -> ResponseBody { + match body { + edgezero_core::body::Body::Once(bytes) => { + http_body_util::Either::Left(http_body_util::Full::new(bytes)) + } + edgezero_core::body::Body::Stream(stream) => { + http_body_util::Either::Right(http_body_util::StreamBody::new( + stream + .map(|chunk| chunk.map(http_body::Frame::data)) + .boxed_local(), + )) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use futures::FutureExt as _; + use http_body_util::BodyExt as _; + + #[tokio::test] + async fn a_streamed_body_reaches_the_host_one_chunk_at_a_time() { + let (sender, receiver) = + futures::channel::mpsc::unbounded::>(); + sender + .unbounded_send(Ok(Bytes::from_static(b"first"))) + .expect("should queue the first chunk"); + let mut body = response_body(edgezero_core::body::Body::from_stream(receiver)); + + let first = body + .frame() + .await + .expect("the first frame should arrive") + .expect("the first frame should be readable") + .into_data() + .expect("the first frame should carry data"); + assert_eq!(first.as_ref(), b"first"); + assert!( + body.frame().now_or_never().is_none(), + "should wait for the next chunk rather than end the body" + ); + + sender + .unbounded_send(Ok(Bytes::from_static(b"second"))) + .expect("should queue the second chunk"); + drop(sender); + let second = body + .frame() + .await + .expect("the second frame should arrive") + .expect("the second frame should be readable") + .into_data() + .expect("the second frame should carry data"); + assert_eq!(second.as_ref(), b"second"); + assert!(body.frame().await.is_none(), "should end with the stream"); + } + + #[tokio::test] + async fn a_failed_chunk_aborts_the_host_body() { + let stream = futures::stream::iter([ + Ok(Bytes::from_static(b"partial")), + Err(std::io::Error::other("origin connection reset")), + ]); + let mut body = response_body(edgezero_core::body::Body::from_stream(stream)); + + body.frame() + .await + .expect("the chunk before the failure should arrive") + .expect("the chunk before the failure should be readable"); + let error = body + .frame() + .await + .expect("the failure should surface") + .expect_err("a failed chunk must not look like the end of the body"); + assert!( + error.to_string().contains("origin connection reset"), + "should carry the failure: {error}" + ); + } + + #[tokio::test] + async fn a_buffered_body_stays_one_buffer() { + let body = response_body(edgezero_core::body::Body::from(b"whole".to_vec())); + + assert!( + matches!(body, http_body_util::Either::Left(_)), + "a buffered router body should not become a stream" + ); + let collected = body + .collect() + .await + .expect("a buffered body should be readable") + .to_bytes(); + assert_eq!(collected.as_ref(), b"whole"); + } } diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 5a553a6328..e16091b385 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -18,6 +18,8 @@ use trusted_server_core::platform::{ #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] use trusted_server_core::platform::UnavailableHttpClient; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use spin_sdk::http::body::IncomingBodyExt as _; #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] use std::io::Read as _; #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] @@ -27,6 +29,9 @@ use trusted_server_core::platform::{ PlatformPendingRequest, PlatformResponse, PlatformSelectResult, }; +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +mod response_stream; + // 8 MiB ceiling: conservative for ad-server responses while leaving headroom in // the Spin WASM component heap. Larger responses from misbehaving origins are // rejected with a typed error rather than OOMing the component. @@ -399,9 +404,22 @@ fn reject_unsupported_request_contracts( return Err(Report::new(PlatformError::HttpClient) .attach("Spin outbound HTTP does not support Image Optimizer metadata")); } + Ok(()) +} + +/// Refuse a streamed response on the buffered `send_async` path. +/// +/// `send_async` stores a completed, buffered response for `select`, so a +/// request that asks for a stream must go through `send` instead of being +/// silently buffered. The check runs before the outbound request is sent. +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] +fn reject_buffered_stream_request( + request: &PlatformHttpRequest, +) -> Result<(), Report> { if request.stream_response { - return Err(Report::new(PlatformError::HttpClient) - .attach("Spin outbound HTTP does not support streaming responses")); + return Err(Report::new(PlatformError::HttpClient).attach( + "Spin outbound HTTP streams responses through send alone; send_async buffers", + )); } Ok(()) } @@ -527,6 +545,15 @@ struct SpinPendingResponse { /// `spin_sdk::http::send`-backed HTTP client for the Spin runtime. /// +/// [`send`](PlatformHttpClient::send) preserves the origin body as a +/// `Body::Stream` when the request asks for it, reading the WASI incoming +/// body one host chunk at a time, see +/// [`response_stream::bounded_origin_stream`]. A streamed body is the +/// origin's bytes as sent, so it keeps its `Content-Encoding` and +/// `Content-Length`, and it honors the caller's exact +/// `max_response_body_bytes`, if any, instead of the buffered path's 8 MiB +/// ceiling. The buffered path still decodes gzip and Brotli bodies. +/// /// `send_async` eagerly executes each request before returning, so /// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false` and the /// auction orchestrator rejects multi-provider configurations before any @@ -557,7 +584,9 @@ impl SpinPlatformHttpClient { request: PlatformHttpRequest, ) -> Result> { reject_unsupported_request_contracts(&request)?; - let response_limit = effective_response_body_limit(request.max_response_body_bytes); + let stream_response = request.stream_response; + let caller_limit = request.max_response_body_bytes; + let response_limit = effective_response_body_limit(caller_limit); let method = request.request.method().clone(); let uri = request.request.uri().to_string(); @@ -616,34 +645,61 @@ impl SpinPlatformHttpClient { .map(|(name, value)| (name.to_string(), value.as_bytes().to_vec())) .collect(); let response_has_body = !response_must_not_have_body(&method, status); + let claimed_len = headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .and_then(|(_, value)| std::str::from_utf8(value).ok()) + .and_then(|value| value.trim().parse::().ok()); + // A streamed body holds one chunk at a time, so only the caller's own + // limit applies to it. A buffered body is also held to the ceiling. + let enforced_limit = if stream_response { + caller_limit + } else { + Some(response_limit) + }; if response_has_body - && headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("content-length")) - .and_then(|(_, value)| std::str::from_utf8(value).ok()) - .and_then(|value| value.trim().parse::().ok()) - .is_some_and(|length| length > response_limit) + && let Some(limit) = enforced_limit + && let Some(claimed_len) = claimed_len + && u64::try_from(limit).is_ok_and(|limit| claimed_len > limit) { return Err(Report::new(PlatformError::HttpClient).attach(format!( - "origin Content-Length exceeds {response_limit}-byte response body limit" + "origin Content-Length exceeds {limit}-byte response body limit" ))); } - let body = if response_has_body { - use spin_sdk::http::body::IncomingBodyExt as _; - collect_bounded_response_stream(spin_response.into_body().stream(), response_limit) - .await? + + let (headers, body) = if stream_response { + let body = if response_has_body { + response_stream::bounded_origin_stream( + spin_response.into_body().stream(), + caller_limit, + claimed_len, + ) + } else { + edgezero_core::body::Body::empty() + }; + (sanitize_response_headers(headers), body) } else { - Vec::new() + let body = if response_has_body { + collect_bounded_response_stream(spin_response.into_body().stream(), response_limit) + .await? + } else { + Vec::new() + }; + let (headers, body) = apply_spin_response_policy_with_limit( + &method, + status, + headers, + body, + response_limit, + )?; + (headers, edgezero_core::body::Body::from(body)) }; - - let (headers, body) = - apply_spin_response_policy_with_limit(&method, status, headers, body, response_limit)?; let mut edge_builder = edgezero_core::http::response_builder().status(status); for (name, value) in headers { edge_builder = edge_builder.header(name.as_str(), value.as_slice()); } let edge_resp = edge_builder - .body(edgezero_core::body::Body::from(body)) + .body(body) .change_context(PlatformError::HttpClient)?; Ok(PlatformResponse::new(edge_resp).with_backend_name(request.backend_name)) @@ -660,6 +716,10 @@ impl PlatformHttpClient for SpinPlatformHttpClient { false } + fn supports_streaming_responses(&self) -> bool { + true + } + async fn send( &self, request: PlatformHttpRequest, @@ -671,6 +731,7 @@ impl PlatformHttpClient for SpinPlatformHttpClient { &self, request: PlatformHttpRequest, ) -> Result> { + reject_buffered_stream_request(&request)?; let backend_name = request.backend_name.clone(); let response = self.execute(request).await?; @@ -1340,12 +1401,20 @@ mod tests { } #[test] - fn unsupported_request_contracts_reject_stream_response() { + fn send_streams_a_response_and_send_async_refuses_to_buffer_one() { let request = platform_request().with_stream_response(); assert!( - reject_unsupported_request_contracts(&request).is_err(), - "Spin must reject streaming-response requests instead of buffering silently" + reject_unsupported_request_contracts(&request).is_ok(), + "send should accept a streamed-response request" + ); + assert!( + reject_buffered_stream_request(&request).is_err(), + "the buffered send_async path must refuse it rather than buffer silently" + ); + assert!( + reject_buffered_stream_request(&platform_request()).is_ok(), + "a buffered request should still use send_async" ); } diff --git a/crates/trusted-server-adapter-spin/src/platform/response_stream.rs b/crates/trusted-server-adapter-spin/src/platform/response_stream.rs new file mode 100644 index 0000000000..df6474af64 --- /dev/null +++ b/crates/trusted-server-adapter-spin/src/platform/response_stream.rs @@ -0,0 +1,306 @@ +//! Origin response bodies passed on as Spin's outbound HTTP delivers them. +//! +//! A streamed response never holds more than the chunk the runtime handed +//! over last. The caller's byte limit and the origin's declared length are +//! checked chunk by chunk, so an oversized, truncated or failed origin body +//! ends the stream with an error instead of reaching the caller as a +//! complete one. + +use std::fmt::Display; +use std::pin::Pin; + +use bytes::Bytes; +use futures::{Stream, StreamExt as _}; + +/// The source and the number of bytes already yielded from it. +struct Progress { + source: Pin>, + yielded: usize, +} + +/// A byte count in the unit of a declared `Content-Length`. A count no +/// `u64` can hold is past any declared length. +fn declared_units(bytes: usize) -> u64 { + u64::try_from(bytes).unwrap_or(u64::MAX) +} + +/// Wrap a runtime body stream as a [`Body::Stream`](edgezero_core::body::Body::Stream). +/// +/// `limit` is the caller's `max_response_body_bytes`. The chunk that would +/// carry the running total past it is replaced by an error, and nothing +/// after it is read. +/// +/// `expected_length` is the origin's `Content-Length`. Spin hands the body +/// over exactly as the origin encoded it, so the declared length describes +/// the bytes this stream yields. A body that ends short of the declared +/// length, or runs past it, ends with an error rather than reaching the +/// caller as a complete one, whatever the host's own framing checks report. +/// Pass `None` when the origin declared no length. +/// +/// A source error is passed on as a stream error and ends the stream. +pub(super) fn bounded_origin_stream( + source: S, + limit: Option, + expected_length: Option, +) -> edgezero_core::body::Body +where + S: Stream> + 'static, + B: Into, + E: Display, +{ + let initial = Progress { + source: Box::pin(source), + yielded: 0, + }; + let stream = futures::stream::unfold(Some(initial), move |state| async move { + let mut progress = state?; + match progress.source.next().await { + None => { + if let Some(expected) = expected_length + && declared_units(progress.yielded) != expected + { + return Some(( + Err(std::io::Error::other(format!( + "Spin outbound response body ended after {} of its declared {expected} bytes", + progress.yielded + ))), + None, + )); + } + None + } + Some(Err(error)) => Some(( + Err(std::io::Error::other(format!( + "failed to read Spin outbound response body: {error}" + ))), + None, + )), + Some(Ok(chunk)) => { + let chunk: Bytes = chunk.into(); + let Some(next_yielded) = progress.yielded.checked_add(chunk.len()) else { + return Some(( + Err(std::io::Error::other( + "Spin outbound response body byte count overflowed", + )), + None, + )); + }; + if let Some(limit) = limit + && next_yielded > limit + { + return Some(( + Err(std::io::Error::other(format!( + "Spin outbound response body exceeded caller limit of {limit} bytes" + ))), + None, + )); + } + if let Some(expected) = expected_length + && declared_units(next_yielded) > expected + { + return Some(( + Err(std::io::Error::other(format!( + "Spin outbound response body ran past its declared {expected} bytes" + ))), + None, + )); + } + progress.yielded = next_yielded; + Some((Ok(chunk), Some(progress))) + } + } + }); + edgezero_core::body::Body::from_stream(stream) +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::FutureExt as _; + + type ChunkResult = Result, std::io::Error>; + + fn into_stream( + body: edgezero_core::body::Body, + ) -> futures::stream::LocalBoxStream<'static, Result> { + body.into_stream() + .expect("a streamed origin body should stay a stream") + } + + #[tokio::test] + async fn each_chunk_is_yielded_before_the_origin_finishes() { + let (sender, receiver) = futures::channel::mpsc::unbounded::(); + sender + .unbounded_send(Ok(b"ab".to_vec())) + .expect("should queue the first chunk"); + let mut stream = into_stream(bounded_origin_stream(receiver, None, None)); + + let first = stream + .next() + .await + .expect("the first chunk should arrive") + .expect("the first chunk should be readable"); + assert_eq!( + first.as_ref(), + b"ab", + "should pass the first chunk on as it arrived" + ); + assert!( + stream.next().now_or_never().is_none(), + "should wait for the origin rather than report an end it has not sent" + ); + + sender + .unbounded_send(Ok(b"cd".to_vec())) + .expect("should queue the second chunk"); + drop(sender); + let second = stream + .next() + .await + .expect("the second chunk should arrive") + .expect("the second chunk should be readable"); + assert_eq!(second.as_ref(), b"cd", "should pass the later chunk on"); + assert!(stream.next().await.is_none(), "should end with the origin"); + } + + #[tokio::test] + async fn an_unbounded_stream_passes_more_than_the_buffered_ceiling() { + // Eleven 1 MiB chunks cross the 8 MiB ceiling the buffered path + // enforces. The chunks share one allocation, so the test holds 1 MiB. + let chunk = Bytes::from(vec![7u8; 1024 * 1024]); + let chunks = std::iter::repeat_n(chunk, 11).map(Ok::<_, std::io::Error>); + let mut stream = into_stream(bounded_origin_stream( + futures::stream::iter(chunks), + None, + None, + )); + + let mut total = 0usize; + while let Some(chunk) = stream.next().await { + total += chunk.expect("an unbounded chunk should be readable").len(); + } + assert_eq!(total, 11 * 1024 * 1024, "should pass every byte on"); + } + + #[tokio::test] + async fn the_chunk_that_crosses_the_caller_limit_ends_the_stream() { + let source = futures::stream::iter([ + Ok::<_, std::io::Error>(b"ab".to_vec()), + Ok(b"cd".to_vec()), + Ok(b"e".to_vec()), + Ok(b"never read".to_vec()), + ]); + let mut stream = into_stream(bounded_origin_stream(source, Some(4), None)); + + for expected in [b"ab", b"cd"] { + let chunk = stream + .next() + .await + .expect("a chunk within the limit should arrive") + .expect("a chunk within the limit should be readable"); + assert_eq!( + chunk.as_ref(), + expected, + "should pass chunks up to the exact limit" + ); + } + let error = stream + .next() + .await + .expect("the crossing chunk should surface") + .expect_err("the crossing chunk must be refused"); + assert!( + error.to_string().contains("limit of 4 bytes"), + "should report the enforced limit: {error}" + ); + assert!( + stream.next().await.is_none(), + "should read nothing after the refused chunk" + ); + } + + #[tokio::test] + async fn an_origin_failure_is_passed_on_and_ends_the_stream() { + let source = futures::stream::iter([ + Ok(b"ab".to_vec()), + Err(std::io::Error::other("connection reset by origin")), + Ok(b"never read".to_vec()), + ]); + let mut stream = into_stream(bounded_origin_stream(source, Some(64), None)); + + let first = stream + .next() + .await + .expect("the chunk before the failure should arrive") + .expect("the chunk before the failure should be readable"); + assert_eq!(first.as_ref(), b"ab"); + let error = stream + .next() + .await + .expect("the failure should surface") + .expect_err("the origin failure must not look like data"); + assert!( + error.to_string().contains("connection reset by origin"), + "should carry the origin's failure: {error}" + ); + assert!(stream.next().await.is_none(), "should end at the failure"); + } + + #[tokio::test] + async fn a_body_cut_short_of_its_declared_length_is_an_error() { + // An origin connection that closes early must not end the body cleanly. + let source = futures::stream::iter([Ok::<_, std::io::Error>(b"ab".to_vec())]); + let mut stream = into_stream(bounded_origin_stream(source, None, Some(4))); + + let first = stream + .next() + .await + .expect("the delivered bytes should arrive") + .expect("the delivered bytes should be readable"); + assert_eq!(first.as_ref(), b"ab"); + let error = stream + .next() + .await + .expect("the early end should surface") + .expect_err("a truncated body must not end cleanly"); + assert!( + error.to_string().contains("2 of its declared 4 bytes"), + "should report how much of the declared body arrived: {error}" + ); + } + + #[tokio::test] + async fn a_body_running_past_its_declared_length_is_an_error() { + let source = + futures::stream::iter([Ok::<_, std::io::Error>(b"abc".to_vec()), Ok(b"de".to_vec())]); + let mut stream = into_stream(bounded_origin_stream(source, None, Some(4))); + + stream + .next() + .await + .expect("the bytes within the declared length should arrive") + .expect("the bytes within the declared length should be readable"); + let error = stream + .next() + .await + .expect("the overrun should surface") + .expect_err("bytes past the declared length must be refused"); + assert!( + error.to_string().contains("declared 4 bytes"), + "should report the declared length: {error}" + ); + } + + #[tokio::test] + async fn a_body_matching_its_declared_length_ends_cleanly() { + let source = + futures::stream::iter([Ok::<_, std::io::Error>(b"ab".to_vec()), Ok(b"cd".to_vec())]); + let mut stream = into_stream(bounded_origin_stream(source, Some(4), Some(4))); + + let mut body = Vec::new(); + while let Some(chunk) = stream.next().await { + body.extend_from_slice(&chunk.expect("a declared-length body should be readable")); + } + assert_eq!(body, b"abcd", "should deliver the whole declared body"); + } +} diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index ec64a217dd..935f0e1aa6 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -29,6 +29,7 @@ futures = { workspace = true } glob = { workspace = true } hex = { workspace = true } hmac = { workspace = true } +html5ever = { workspace = true } http = { workspace = true } httpdate = { workspace = true } jose-jwk = { workspace = true } diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 962001fc22..39753700c3 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -8,6 +8,8 @@ use trusted_server_core::streaming_processor::StreamProcessor as _; fn make_config() -> HtmlProcessorConfig { HtmlProcessorConfig { csp_nonce_observed: None, + rehost: trusted_server_core::rehost::PageRehost::default(), + response_headers: None, public_prefix: None, origin_host: "origin.bench.example.com".to_string(), request_host: "proxy.bench.example.com".to_string(), diff --git a/crates/trusted-server-core/src/asset_image_optimizer.rs b/crates/trusted-server-core/src/asset_image_optimizer.rs index bbc2393fb4..971c0ecf47 100644 --- a/crates/trusted-server-core/src/asset_image_optimizer.rs +++ b/crates/trusted-server-core/src/asset_image_optimizer.rs @@ -418,6 +418,7 @@ mod tests { region: "us_east".to_string(), profile_set: "default_images".to_string(), origin_query: None, + signed_source: false, }); let err = options_for_asset_request(&settings, &route, "") diff --git a/crates/trusted-server-core/src/creative.rs b/crates/trusted-server-core/src/creative.rs index e37093b2f3..235f3e627c 100644 --- a/crates/trusted-server-core/src/creative.rs +++ b/crates/trusted-server-core/src/creative.rs @@ -180,12 +180,71 @@ fn rewrite_style_urls_in_context( base_origin: &str, allows_import_rules: bool, output_limit: usize, +) -> Result { + rewrite_css_urls( + &mut CreativeProxy { + settings, + base_origin, + }, + style, + allows_import_rules, + output_limit, + ) +} + +/// Where in a stylesheet a URL was read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CssUrlContext { + /// The stylesheet an `@import` loads. + Import, + /// Anything else the browser fetches: an image, a font, a cursor. + Resource, +} + +/// Decides what each URL a stylesheet names becomes, as +/// [`rewrite_css_urls`] reads them. +pub(crate) trait CssUrlPolicy { + /// The address to write in place of `value`, the URL as the browser reads + /// it with its escapes resolved, or `None` to keep its original bytes. + fn replace(&mut self, value: &str, context: CssUrlContext) -> Option; +} + +/// The creative policy: an absolute or protocol-relative address goes +/// through the signed first-party proxy, see [`build_proxy_url`]. +struct CreativeProxy<'a> { + settings: &'a Settings, + base_origin: &'a str, +} + +impl CssUrlPolicy for CreativeProxy<'_> { + fn replace(&mut self, value: &str, _context: CssUrlContext) -> Option { + let absolute = to_abs(self.settings, value)?; + Some(build_proxy_url(self.settings, &absolute, self.base_origin)) + } +} + +/// Rewrites every URL `style` names as `policy` decides, within the nesting +/// and output-size limits, keeping every other byte. A reference keeps the +/// shape it was read in, see [`rewrite_style_urls`]. +/// +/// `allows_import_rules` says whether `style` is a stylesheet, where +/// `@import` can begin a rule, rather than a style attribute. +/// +/// # Errors +/// +/// When the CSS nests past [`MAX_CSS_NESTING_DEPTH`] or the output would be +/// larger than `output_limit`. +pub(crate) fn rewrite_css_urls( + policy: &mut dyn CssUrlPolicy, + style: &str, + allows_import_rules: bool, + output_limit: usize, ) -> Result { let mut rewriter = CssUrlRewriter { - settings, + policy, style, - base_origin, allows_import_rules, + in_import: false, out: CssOutput { value: String::with_capacity(style.len().min(output_limit)), limit: output_limit, @@ -229,13 +288,15 @@ impl fmt::Write for CssOutput { } } -/// Splices rewritten `url()` values into a copy of the original CSS. +/// Splices the URLs a policy replaces into a copy of the original CSS. struct CssUrlRewriter<'a> { - settings: &'a Settings, + policy: &'a mut dyn CssUrlPolicy, style: &'a str, - base_origin: &'a str, /// Whether the outermost scope is a stylesheet that may contain `@import`. allows_import_rules: bool, + /// Whether the walk is inside an `@import` prelude, whose URL names a + /// stylesheet. + in_import: bool, out: CssOutput, write_pos: usize, /// Any refusal invalidates the entire rewrite. @@ -340,7 +401,9 @@ impl CssUrlRewriter<'_> { let _ = parser.parse_until_before( cssparser::Delimiter::Semicolon | cssparser::Delimiter::CurlyBracketBlock, |prelude| -> Result<(), cssparser::ParseError<'_, ()>> { + let outer = std::mem::replace(&mut self.in_import, true); self.descend(prelude, depth, BareStringUrls::FirstValue); + self.in_import = outer; Ok(()) }, ); @@ -390,8 +453,8 @@ impl CssUrlRewriter<'_> { self.walk(parser, depth + 1, strings_are_urls); } - /// Replaces the span just consumed with a proxied reference in `shape`, or - /// leaves it. + /// Replaces the span just consumed with the policy's address in `shape`, + /// or leaves it. fn rewrite( &mut self, value: &str, @@ -399,14 +462,19 @@ impl CssUrlRewriter<'_> { parser: &cssparser::Parser<'_, '_>, shape: UrlShape, ) { - let Some(absolute) = to_abs(self.settings, value) else { + let context = if self.in_import { + CssUrlContext::Import + } else { + CssUrlContext::Resource + }; + let Some(replacement) = self.policy.replace(value, context) else { return; }; let token_end = parser.position().byte_index(); let result = (|| -> fmt::Result { self.out .write_str(&self.style[self.write_pos..token_start])?; - let proxied = build_proxy_url(self.settings, &absolute, self.base_origin); + let proxied = replacement; if let UrlShape::Function(name) = shape { self.out.write_str(name)?; self.out.write_char('(')?; diff --git a/crates/trusted-server-core/src/css_relocation.rs b/crates/trusted-server-core/src/css_relocation.rs new file mode 100644 index 0000000000..c0386421ca --- /dev/null +++ b/crates/trusted-server-core/src/css_relocation.rs @@ -0,0 +1,231 @@ +//! A stylesheet a rehosting route serves from the reader's host, its own +//! addresses moved with it and its `[[serve]]` entry run on it, see +//! [`crate::rehost`]. +//! +//! A stylesheet's relative and root-relative addresses resolve against the +//! address it is fetched from, so serving it from another address changes +//! what they name. Every `url()`, `src()`, `image-set()` candidate and +//! `@import` the browser would fetch is read with core's CSS tokenizer, +//! resolved against the stylesheet's upstream address, and written as the +//! address that fetches the same thing: a rehosting route's address where +//! one serves it, the absolute upstream address otherwise. A fragment-only +//! reference and `data:`, `blob:` and `about:` addresses are left as written. +//! An `@import` moves to a route only where that route moves stylesheets. + +use bytes::Bytes; +use error_stack::Report; +use http::Method; +use url::Url; + +use crate::cache_policy::{CachePolicy, CacheVisibility}; +use crate::creative::{CssUrlContext, CssUrlPolicy, rewrite_css_urls}; +use crate::error::TrustedServerError; +use crate::integrations::{IntegrationRegistry, IntegrationRequestState}; +use crate::media_signing::OriginFreshness; +use crate::middleware::{MiddlewarePhase, Prepared, ResponseHeader}; +use crate::module_context::{ModuleContext, ModuleRequest}; +use crate::platform::RuntimeServices; +use crate::rehost::{PageAddress, Reference, RehostRoutes}; +use crate::settings::Settings; +use crate::streaming_processor::StreamProcessor as _; + +/// The largest stylesheet a rehosting route moves, before and after its +/// addresses are moved. +pub(crate) const MAX_STYLESHEET_BYTES: usize = 10 * 1024 * 1024; + +/// Keep buffered bytes without copying. Streamed chunks are checked against +/// the remaining capacity by the shared upstream collector before appending. +pub(crate) async fn collect_stylesheet( + body: edgezero_core::body::Body, +) -> Result> { + match body { + edgezero_core::body::Body::Once(bytes) if bytes.len() <= MAX_STYLESHEET_BYTES => Ok(bytes), + body => crate::integrations::collect_response_bounded( + body, + MAX_STYLESHEET_BYTES, + "stylesheet_relocation", + ) + .await + .map(Bytes::from), + } +} + +/// A relocated answer cannot outlive its source or the route's policy. +/// Stale allowances and immutability do not transfer to these mutable URLs. +pub(crate) fn bounded_cache_policy( + remaining: OriginFreshness, + route: Option, +) -> CachePolicy { + CachePolicy { + visibility: route.map_or(CacheVisibility::Public, |policy| policy.visibility), + browser_ttl: Some( + route + .and_then(|policy| policy.browser_ttl) + .map_or(remaining.browser, |cap| remaining.browser.min(cap)), + ), + edge_ttl: Some( + route + .and_then(|policy| policy.edge_ttl) + .map_or(remaining.shared, |cap| remaining.shared.min(cap)), + ), + stale_while_revalidate: None, + stale_if_error: None, + immutable: false, + } +} + +/// The route policy: each address resolved against the stylesheet's +/// upstream address and written as the address that fetches the same thing. +struct RouteAddresses<'a> { + stylesheet: &'a Url, + routes: &'a RehostRoutes, + page: PageAddress<'a>, +} + +impl CssUrlPolicy for RouteAddresses<'_> { + fn replace(&mut self, value: &str, context: CssUrlContext) -> Option { + let value = value.trim(); + if value.is_empty() || value.starts_with('#') { + return None; + } + let resolved = self.stylesheet.join(value).ok()?; + if !matches!(resolved.scheme(), "http" | "https") { + return None; + } + let reference = match context { + CssUrlContext::Import => Reference::Style, + CssUrlContext::Resource => Reference::Media, + }; + Some( + self.routes + .rehosted(&resolved, reference, &self.page) + .unwrap_or_else(|| resolved.to_string()), + ) + } +} + +/// Why a rehosted stylesheet goes to the reader at its upstream address +/// instead. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub(crate) enum NotRelocated { + /// The body is not UTF-8, or names another charset with `@charset`. + #[display("stylesheet_not_utf8")] + NotUtf8, + /// The tokenizer refused it: nested too deep or grown too large. + #[display("stylesheet_not_rewritable")] + NotRewritable, + /// A middleware of its `[[serve]]` entry refused it. + #[display("stylesheet_refused")] + Refused, +} + +/// One stylesheet a rehosting route serves. +pub(crate) struct Stylesheet<'a> { + pub settings: &'a Settings, + pub services: &'a RuntimeServices, + pub integration_registry: &'a IntegrationRegistry, + /// The address the stylesheet was fetched from. + pub upstream: &'a Url, + /// Its `Content-Type`. + pub content_type: &'a str, + /// The route path the reader asked for, after any prefix request + /// addressing removed. + pub route_path: &'a str, + pub page: PageAddress<'a>, + /// The headers of the reader's response, as the middleware are told + /// them. + pub response_headers: &'a http::HeaderMap, +} + +impl Stylesheet<'_> { + /// `body` with its addresses moved and its `[[serve]]` entry's + /// middleware run on it, with the headers those middleware write. + /// + /// # Errors + /// + /// Why the stylesheet cannot be served from the route. + pub(crate) async fn relocate( + &self, + body: &Bytes, + ) -> Result<(Vec, Vec), NotRelocated> { + let text = std::str::from_utf8(body).map_err(|_| NotRelocated::NotUtf8)?; + if names_another_charset(text) { + return Err(NotRelocated::NotUtf8); + } + let routes = RehostRoutes::of(self.settings); + let relocated = rewrite_css_urls( + &mut RouteAddresses { + stylesheet: self.upstream, + routes: &routes, + page: self.page, + }, + text, + true, + MAX_STYLESHEET_BYTES, + ) + .map_err(|_| NotRelocated::NotRewritable)?; + + let prepared = Prepared::default(); + let request_state = IntegrationRequestState::default(); + let document = crate::publisher::TextResponse { + content_type: self.content_type, + status: http::StatusCode::OK, + origin_private: false, + response_headers: Some(self.response_headers), + may_be_stored: false, + request_path: self.route_path, + request_host: self.page.host, + request_scheme: self.page.scheme, + origin_host: self.upstream.host_str().unwrap_or_default(), + request_state: &request_state, + max_buffered_script_bytes: self.settings.publisher.max_buffered_body_bytes, + prepared: &prepared, + }; + // Served to every reader alike, so prepared with no reader's state. + let services = self.services.clone().without_connection(); + let removed_prefix = + (!self.page.removed_prefix.is_empty()).then_some(self.page.removed_prefix); + let method = Method::GET; + let context = ModuleContext::new( + ModuleRequest::new(&method, self.page.host, self.page.scheme, self.route_path) + .with_removed_prefix(removed_prefix), + ) + .with_settings(self.settings) + .with_services(&services); + crate::publisher::prepare_document( + &document, + &[(MiddlewarePhase::Serve, &context)], + self.settings, + self.integration_registry, + ) + .await + .map_err(|_| NotRelocated::Refused)?; + let (mut processor, headers) = crate::publisher::middleware_stream_processors( + &document, + &[MiddlewarePhase::Serve], + self.settings, + self.integration_registry, + ) + .map_err(|_| NotRelocated::Refused)? + .into_middleware_processor(); + let signed = processor + .finish(relocated.as_bytes(), self.services) + .await + .map_err(|_| NotRelocated::Refused)?; + Ok((signed, headers)) + } +} + +/// Whether a stylesheet's leading `@charset` names an encoding other than +/// UTF-8. +fn names_another_charset(text: &str) -> bool { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let Some(rest) = text.strip_prefix("@charset \"") else { + return false; + }; + let name = rest.split('"').next().unwrap_or_default(); + !name.eq_ignore_ascii_case("utf-8") +} + +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/css_relocation/tests.rs b/crates/trusted-server-core/src/css_relocation/tests.rs new file mode 100644 index 0000000000..a7c5614b43 --- /dev/null +++ b/crates/trusted-server-core/src/css_relocation/tests.rs @@ -0,0 +1,211 @@ +use super::*; +use crate::middleware::{PhaseEntries, PhaseEntry}; +use crate::settings::ProxyAssetRoute; +use crate::test_support::tests::create_test_settings; +use edgezero_core::body::Body as EdgeBody; +use std::time::Duration; + +const ROUTE: &str = "/_rehost/cdn.example.net/npm/"; + +fn settings_rehosting(stylesheets_under: &str) -> Settings { + let mut settings = create_test_settings(); + let mut route = ProxyAssetRoute::new(ROUTE, "https://cdn.example.net/npm"); + route.rehost_page_references = true; + settings.proxy.asset_routes.push(route); + settings.serve = PhaseEntries::new(vec![PhaseEntry { + media_type: "text/css".to_owned(), + path: Some(format!("{ROUTE}{stylesheets_under}")), + middleware: vec!["provenance.encypher.stylesheet".to_owned()], + }]); + settings +} + +fn relocated(settings: &Settings, css: &str) -> String { + let routes = RehostRoutes::of(settings); + let stylesheet = + Url::parse("https://cdn.example.net/npm/slick@1/slick/theme.css?ver=1").expect("url"); + rewrite_css_urls( + &mut RouteAddresses { + stylesheet: &stylesheet, + routes: &routes, + page: PageAddress { + scheme: "https", + host: "reader.example.com", + removed_prefix: "/www.publisher.example", + base: None, + }, + }, + css, + true, + MAX_STYLESHEET_BYTES, + ) + .expect("should rewrite") +} + +#[test] +fn a_moved_stylesheet_s_addresses_keep_naming_what_they_named() { + let css = concat!( + "@import \"../shared/base.css\" layer(theme);\n", + ".a{background:url(./ajax-loader.gif)}\n", + "@font-face{src:url(\"fonts/slick.woff\") format(\"woff\"),url('fonts/slick.svg#slick')}\n", + ".b{background:url(/root/x.png)}\n", + ".c{background:image-set(\"https://elsewhere.example/a.png\" 1x)}\n", + ".d{filter:url(#blur);background:url(data:image/png;base64,AAAA)}\n", + ".e{background:url(\"fonts\\2f esc.png\")}\n", + ); + let out = relocated(&settings_rehosting("slick@1/shared/"), css); + let route = format!("https://reader.example.com/www.publisher.example{ROUTE}"); + for expected in [ + // An approved stylesheet path moves with the route. + format!("@import \"{route}slick@1/shared/base.css\" layer(theme);"), + format!("url(\"{route}slick@1/slick/ajax-loader.gif\")"), + format!("url(\"{route}slick@1/slick/fonts/slick.woff\")"), + format!("url(\"{route}slick@1/slick/fonts/slick.svg#slick\")"), + format!("url(\"{route}slick@1/slick/fonts/esc.png\")"), + // Outside the folder, so the absolute upstream address. + "url(\"https://cdn.example.net/root/x.png\")".to_owned(), + "\"https://elsewhere.example/a.png\" 1x".to_owned(), + // Document-local and inline references stay as written. + "url(#blur)".to_owned(), + "url(data:image/png;base64,AAAA)".to_owned(), + ] { + assert!(out.contains(&expected), "should contain {expected}: {out}"); + } +} + +#[test] +fn an_import_no_entry_moves_keeps_its_upstream_address() { + let out = relocated( + &settings_rehosting("other/"), + "@import url(\"../shared/base.css\");", + ); + assert!( + out.contains("url(\"https://cdn.example.net/npm/slick@1/shared/base.css\")"), + "a stylesheet no entry moves is fetched where it lives: {out}" + ); +} + +#[test] +fn a_stylesheet_naming_another_charset_is_not_moved() { + assert!(names_another_charset("@charset \"iso-8859-1\";\n.a{}")); + assert!(!names_another_charset("@charset \"UTF-8\";\n.a{}")); + assert!(!names_another_charset(".a{}")); +} + +#[test] +fn relocated_cache_lifetimes_are_bounded_by_both_origin_and_route() { + let route = CachePolicy { + browser_ttl: Some(Duration::from_secs(60)), + edge_ttl: Some(Duration::from_secs(600)), + ..CachePolicy::public_immutable(Duration::from_secs(60)) + }; + for (browser, shared, expected_browser, expected_shared) in [ + (300, 3600, 60, 600), + (20, 300, 20, 300), + (20, 3600, 20, 600), + (300, 20, 60, 20), + (0, 300, 0, 300), + (20, 0, 20, 0), + ] { + let policy = bounded_cache_policy( + OriginFreshness { + browser: Duration::from_secs(browser), + shared: Duration::from_secs(shared), + }, + Some(route), + ); + assert_eq!( + policy.browser_ttl, + Some(Duration::from_secs(expected_browser)) + ); + assert_eq!(policy.edge_ttl, Some(Duration::from_secs(expected_shared))); + assert!(!policy.immutable); + assert_eq!(policy.stale_while_revalidate, None); + assert_eq!(policy.stale_if_error, None); + } + let policy = bounded_cache_policy( + OriginFreshness { + browser: Duration::from_secs(20), + shared: Duration::from_secs(300), + }, + None, + ); + assert_eq!(policy.browser_ttl, Some(Duration::from_secs(20))); + assert_eq!(policy.edge_ttl, Some(Duration::from_secs(300))); +} + +#[test] +fn buffered_stylesheets_keep_their_allocation_up_to_the_bound() { + futures::executor::block_on(async { + for length in [0, 16, MAX_STYLESHEET_BYTES] { + let bytes = Bytes::from(vec![b' '; length]); + let pointer = bytes.as_ptr(); + let collected = collect_stylesheet(EdgeBody::Once(bytes)) + .await + .expect("bounded buffered stylesheet"); + assert_eq!(collected.len(), length); + assert_eq!(collected.as_ptr(), pointer, "buffered bytes stay zero-copy"); + } + assert!( + collect_stylesheet(EdgeBody::Once(Bytes::from(vec![ + 0; + MAX_STYLESHEET_BYTES + 1 + ]))) + .await + .is_err(), + "oversized buffered bytes are refused" + ); + }); +} + +#[test] +fn streamed_stylesheets_refuse_oversized_chunks_without_reading_on() { + futures::executor::block_on(async { + for first_length in [0, MAX_STYLESHEET_BYTES - 1] { + let body = EdgeBody::from_stream(async_stream::stream! { + if first_length != 0 { + yield Ok::(Bytes::from(vec![0; first_length])); + } + // Includes both a single oversized chunk and a chunk larger + // than the remaining capacity after a valid prefix. + let length = if first_length == 0 { MAX_STYLESHEET_BYTES + 1 } else { 2 }; + yield Ok(Bytes::from(vec![0; length])); + panic!("must refuse the oversized chunk before reading another"); + }); + let error = collect_stylesheet(body) + .await + .expect_err("must refuse excess"); + assert!(matches!( + error.current_context(), + TrustedServerError::Integration { .. } + )); + } + }); +} + +#[test] +fn streamed_stylesheets_accept_the_exact_bound_and_report_upstream_errors() { + futures::executor::block_on(async { + let body = EdgeBody::stream(futures::stream::iter([ + Bytes::from(vec![b'a'; MAX_STYLESHEET_BYTES - 1]), + Bytes::from_static(b"b"), + ])); + let bytes = collect_stylesheet(body) + .await + .expect("exactly bounded stream"); + assert_eq!(bytes.len(), MAX_STYLESHEET_BYTES); + assert_eq!(bytes.last(), Some(&b'b')); + + let body = EdgeBody::from_stream(futures::stream::iter([ + Ok(Bytes::from_static(b"partial")), + Err(std::io::Error::other("origin reset")), + ])); + let error = collect_stylesheet(body) + .await + .expect_err("upstream failure"); + assert!(matches!( + error.current_context(), + TrustedServerError::Integration { .. } + )); + }); +} diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 44929b7825..eb9ea6cbce 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -1,6 +1,7 @@ //! Simplified HTML processor that combines URL replacement and integration injection //! //! This module provides a `StreamProcessor` implementation for HTML content. +use std::borrow::Cow; use std::cell::{Cell, RefCell}; use std::io; use std::rc::Rc; @@ -27,13 +28,16 @@ use crate::middleware::{ use crate::platform::RuntimeServices; use crate::prefix::{PUBLIC_PREFIX_ATTRIBUTE, PublicPrefix}; use crate::publisher::build_empty_bids_script; +use crate::rehost::{Decision, PageAddress, PageRehost, Reference}; use crate::settings::Settings; use crate::streaming_processor::{HtmlRewriterAdapter, StreamProcessor}; use crate::tsjs; mod script_text; +mod source_protection; use script_text::ScriptTextGuard; +use source_protection::SourceProtected; /// The HTML rewriter followed by the middleware's stream processors, each /// handed what the one before it produced. @@ -152,6 +156,13 @@ pub struct HtmlProcessorConfig { pub status: StatusCode, /// See [`MiddlewareContext::origin_private`]. pub origin_private: bool, + /// The reader's response headers, see + /// [`MiddlewareContext::response_headers`]. `None` where the document may + /// be stored and replayed to other readers under other headers. + pub response_headers: Option>, + /// The routes that rehost third-party addresses on this page, and the + /// prefix the page's own address carries, see [`crate::rehost`]. + pub rehost: crate::rehost::PageRehost, } impl HtmlProcessorConfig { @@ -180,6 +191,12 @@ impl HtmlProcessorConfig { prepared: Arc::new(Prepared::default()), status: StatusCode::OK, origin_private: false, + response_headers: None, + rehost: crate::rehost::PageRehost { + routes: crate::rehost::RehostRoutes::of(settings), + removed_prefix: String::new(), + base: None, + }, } } @@ -201,6 +218,14 @@ impl HtmlProcessorConfig { self } + /// Hands the serve chain, and the fetch chain of a document that is not + /// stored, the headers of the reader's response. + #[must_use] + pub fn with_response_headers(mut self, headers: Option>) -> Self { + self.response_headers = headers; + self + } + /// Attach the head script carrying this request's permission state. /// /// Separate from [`with_ad_state`](Self::with_ad_state) because the two are @@ -247,6 +272,24 @@ impl HtmlProcessorConfig { self } + /// Sets the path prefix request addressing removed from the page's + /// address, which a rehosted address is written under, see + /// [`crate::rehost`]. + #[must_use] + pub fn with_removed_prefix(mut self, removed_prefix: &str) -> Self { + removed_prefix.clone_into(&mut self.rehost.removed_prefix); + self + } + + /// Sets the effective base resolved from the complete original document. + /// Publisher callers must perform that bounded prepass before streaming + /// any rewritten HTML. + #[must_use] + pub fn with_document_base(mut self, base: url::Url) -> Self { + self.rehost.base = Some(base); + self + } + /// Attach what the middleware prepared for this document. #[must_use] pub fn with_prepared(mut self, prepared: Arc) -> Self { @@ -333,6 +376,24 @@ pub fn create_html_processor_with_middleware( fetch: &MiddlewareChain, serve: Option<&MiddlewareChain>, ) -> Result<(impl StreamProcessor + use<>, Vec), String> { + let mut protection = vec![fetch]; + protection.extend(serve); + create_html_processor_with_source_protection(config, fetch, serve, &protection) +} + +/// Builds the HTML pass with original-source protection selected from both +/// applicable phases, even when serve actions are deferred until cache replay. +/// +/// # Errors +/// +/// Returns the same planning errors as [`create_html_processor_with_middleware`]. +pub fn create_html_processor_with_source_protection( + config: HtmlProcessorConfig, + fetch: &MiddlewareChain, + serve: Option<&MiddlewareChain>, + protection: &[&MiddlewareChain], +) -> Result<(impl StreamProcessor + use<>, Vec), String> { + let limit = config.max_buffered_body_bytes; let document_state = IntegrationDocumentState::default(); let plan = fetch.plan(&MiddlewareContext { phase: fetch.phase(), @@ -341,13 +402,16 @@ pub fn create_html_processor_with_middleware( origin_host: &config.origin_host, status: config.status, origin_private: config.origin_private, + // A document a serve chain is not run with may be stored, so its fetch + // chain is told nothing of this reader's headers. + response_headers: serve.and(config.response_headers.as_deref()), platform: config.integrations.platform(), document_state: &document_state, max_buffered_script_bytes: config.max_buffered_body_bytes, prepared: &config.prepared, })?; let reader = match serve { - Some(serve) => create_serve_processor( + Some(serve) => create_serve_processor_unprotected( serve, &ReaderDocument { request_host: &config.request_host, @@ -355,6 +419,7 @@ pub fn create_html_processor_with_middleware( origin_host: &config.origin_host, status: config.status, origin_private: config.origin_private, + response_headers: config.response_headers.as_deref(), platform: config.integrations.platform(), request_state: &config.request_state, max_buffered_script_bytes: config.max_buffered_body_bytes, @@ -365,7 +430,10 @@ pub fn create_html_processor_with_middleware( }; let mut processor = build_html_processor(config, plan); processor.processors.extend(reader.processor); - Ok((processor, reader.response_headers)) + Ok(( + SourceProtected::new(processor, protection, limit), + reader.response_headers, + )) } /// What a serve chain is told about the reader's copy it works on. @@ -382,6 +450,8 @@ pub struct ReaderDocument<'a> { pub status: StatusCode, /// See [`MiddlewareContext::origin_private`]. pub origin_private: bool, + /// See [`MiddlewareContext::response_headers`]. + pub response_headers: Option<&'a http::HeaderMap>, /// What the adapter reports about the platform the deployment runs on. pub platform: &'a PlatformIdentity, /// What modules left on this reader's request for their page changes. @@ -407,6 +477,38 @@ pub struct ReaderDocument<'a> { pub fn create_serve_processor( chain: &MiddlewareChain, document: &ReaderDocument<'_>, +) -> Result { + create_serve_processor_with_source_protection(chain, document, &[chain]) +} + +/// Protects a cached reader copy using both applicable phases' source policies. +/// +/// # Errors +/// +/// Returns the same planning errors as [`create_serve_processor`]. +pub fn create_serve_processor_with_source_protection( + chain: &MiddlewareChain, + document: &ReaderDocument<'_>, + protection: &[&MiddlewareChain], +) -> Result { + let mut pass = create_serve_processor_unprotected(chain, document)?; + if let Some(processor) = pass.processor.take() { + let inner = HtmlWithStreamingProcessors { + inner: processor, + processors: Vec::new(), + }; + pass.processor = Some(Box::new(SourceProtected::new( + inner, + protection, + document.max_buffered_script_bytes, + ))); + } + Ok(pass) +} + +fn create_serve_processor_unprotected( + chain: &MiddlewareChain, + document: &ReaderDocument<'_>, ) -> Result { if chain.is_empty() { return Ok(ServePass::default()); @@ -420,6 +522,7 @@ pub fn create_serve_processor( origin_host: document.origin_host, status: document.status, origin_private: document.origin_private, + response_headers: document.response_headers, platform: document.platform, document_state: &document_state, max_buffered_script_bytes: document.max_buffered_script_bytes, @@ -457,6 +560,56 @@ type ContentHandler = ( lol_html::ElementContentHandlers<'static>, ); +/// The same structural safety observation applies before rewriting and after +/// restoring protected ranges. Text mentioning nonce attributes is not markup. +fn markup_nonce_handlers(observed: Arc) -> Vec { + let meta_observed = Arc::clone(&observed); + vec![ + element!("meta[http-equiv][content]", move |el| { + let delivers_csp = el.get_attribute("http-equiv").is_some_and(|equiv| { + matches!( + equiv.trim().to_ascii_lowercase().as_str(), + "content-security-policy" | "content-security-policy-report-only" + ) + }); + if delivers_csp + && el + .get_attribute("content") + .is_some_and(|policy| policy.to_ascii_lowercase().contains("'nonce-")) + { + meta_observed.store(true, Ordering::SeqCst); + } + Ok(()) + }), + // lol_html does not decode quoted meta content. Observe nonce attributes + // independently so encoded policies cannot hide nonce-bound elements. + element!("[nonce]", move |_el| { + observed.store(true, Ordering::SeqCst); + Ok(()) + }), + ] +} + +/// Observe fully restored markup before shared-template admission, including +/// protected originals and middleware-generated elements. Never rewrite it. +pub(crate) fn restored_markup_has_nonce(bytes: &[u8]) -> Result { + let observed = Arc::new(AtomicBool::new(false)); + let mut parser = lol_html::HtmlRewriter::new( + RewriterSettings { + element_content_handlers: markup_nonce_handlers(Arc::clone(&observed)), + ..RewriterSettings::default() + }, + |_: &[u8]| {}, + ); + parser + .write(bytes) + .map_err(|error| io::Error::other(error.to_string()))?; + parser + .end() + .map_err(|error| io::Error::other(error.to_string()))?; + Ok(observed.load(Ordering::SeqCst)) +} + /// The handlers that carry out a plan's element and text handlers. /// /// The text handlers share `edits` with every text handler registered before @@ -596,6 +749,117 @@ fn build_serve_processor(plan: MiddlewarePlan) -> HtmlWithStreamingProcessors { } } +/// Decodes an HTML attribute with the tokenizer's attribute-value rules. +/// Only the attribute is tokenized; the document is never copied. +fn decode_html_attribute(value: &str) -> Cow<'_, str> { + if !value.contains(['&', '\r', '\0']) { + return Cow::Borrowed(value); + } + use html5ever::tokenizer::{ + BufferQueue, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts, + }; + #[derive(Default)] + struct AttributeValue(RefCell); + impl TokenSink for AttributeValue { + type Handle = (); + + fn process_token(&self, token: Token, _: u64) -> TokenSinkResult<()> { + if let Token::TagToken(tag) = token + && let Some(attribute) = tag.attrs.into_iter().next() + { + *self.0.borrow_mut() = attribute.value.to_string(); + } + TokenSinkResult::Continue + } + } + // Protect literal quotes, leaving references to be decoded exactly once. + let input = BufferQueue::default(); + input.push_back(format!("", value.replace('"', """)).into()); + let tokenizer = Tokenizer::new(AttributeValue::default(), TokenizerOpts::default()); + let _ = tokenizer.feed(&input); + tokenizer.end(); + Cow::Owned(tokenizer.sink.0.into_inner()) +} + +/// `lol_html` escapes quotes but leaves ampersands alone. Serialize a decoded +/// URL once so its literal ampersands cannot create new HTML references. +fn url_attribute(value: String) -> String { + if value.contains('&') { + value.replace('&', "&") + } else { + value + } +} + +/// Reads the first effective HTML base from complete original bytes. The +/// parser discards output and builds no DOM. Template contents and foreign +/// base elements do not set the document's base. +pub(crate) fn effective_document_base( + original: &[u8], + document_url: &url::Url, +) -> Result { + let base = Rc::new(RefCell::new(document_url.clone())); + let seen = Rc::new(Cell::new(false)); + let template_depth = Rc::new(Cell::new(0usize)); + let handlers = vec![ + element!("template", { + let depth = Rc::clone(&template_depth); + move |el| { + if el.namespace_uri() == "http://www.w3.org/1999/xhtml" { + depth.set(depth.get() + 1); + let depth = Rc::clone(&depth); + if let Some(handlers) = el.end_tag_handlers() { + let handler: EndTagHandler<'static> = Box::new(move |_| { + depth.set(depth.get().saturating_sub(1)); + Ok(()) + }); + handlers.push(handler); + } + } + Ok(()) + } + }), + element!("base[href]", { + let base = Rc::clone(&base); + let seen = Rc::clone(&seen); + let template_depth = Rc::clone(&template_depth); + move |el| { + if el.namespace_uri() != "http://www.w3.org/1999/xhtml" + || template_depth.get() != 0 + || seen.replace(true) + { + return Ok(()); + } + if let Some(href) = el.get_attribute("href") { + let resolved = base.borrow().join(&decode_html_attribute(&href)); + if let Ok(resolved) = resolved + && !matches!(resolved.scheme(), "data" | "javascript") + { + base.replace(resolved); + } + } + Ok(()) + } + }), + ]; + let mut parser = lol_html::HtmlRewriter::new( + RewriterSettings { + element_content_handlers: handlers, + ..RewriterSettings::default() + }, + |_: &[u8]| {}, + ); + parser + .write(original) + .map_err(|error| io::Error::other(error.to_string()))?; + parser + .end() + .map_err(|error| io::Error::other(error.to_string()))?; + Ok(Rc::try_unwrap(base) + .expect("base parser released its handlers") + .into_inner()) +} + /// `value` with each occurrence of a form's text replaced by that form's /// replacement, in one pass from the start, so no replacement is read again. /// @@ -624,6 +888,104 @@ fn replace_each_once(value: &str, forms: &[(String, String)]) -> String { out } +/// What an element's address in `attribute` is fetched as, where a +/// rehosting route may serve it, or `None` for an address no rehosting +/// route moves, such as a link a reader follows. `tag` is the element's +/// name in lower case; `rel`, `link_as` and `input_type` are its `rel`, `as` +/// and `type` attributes. +fn rehost_reference( + tag: &str, + attribute: &str, + rel: Option<&str>, + link_as: Option<&str>, + input_type: Option<&str>, +) -> Option { + let has_rel = |wanted: &str| { + rel.is_some_and(|rel| { + rel.split_ascii_whitespace() + .any(|token| token.eq_ignore_ascii_case(wanted)) + }) + }; + match (tag, attribute) { + ("link", "href") if has_rel("stylesheet") => Some(Reference::Style), + ("link", "href") if has_rel("preload") || has_rel("prefetch") => { + match link_as + .map(|value| value.trim().to_ascii_lowercase()) + .as_deref() + { + Some("style") => Some(Reference::Style), + Some("image" | "font" | "audio" | "video" | "track") => Some(Reference::Media), + _ => None, + } + } + ("link", "imagesrcset") + | ("img" | "source" | "video" | "audio" | "track", "src") + | ("img" | "source", "srcset") + | ("video", "poster") => Some(Reference::Media), + ("input", "src") + if input_type.is_some_and(|value| value.trim().eq_ignore_ascii_case("image")) => + { + Some(Reference::Media) + } + _ => None, + } +} + +/// What the page's rehosting routes decide about `value`, an element's +/// address in `attribute`, see [`crate::rehost::RehostRoutes::decide`]. +fn rehost_decision( + rehost: &PageRehost, + page: &PageAddress<'_>, + el: &lol_html::html_content::Element<'_, '_>, + attribute: &str, + value: &str, +) -> Decision { + if rehost.routes.is_empty() { + return Decision::Unmatched; + } + let tag = el.tag_name().to_ascii_lowercase(); + let rel = el.get_attribute("rel"); + let link_as = el.get_attribute("as"); + let input_type = el.get_attribute("type"); + let Some(reference) = rehost_reference( + &tag, + attribute, + rel.as_deref(), + link_as.as_deref(), + input_type.as_deref(), + ) else { + return Decision::Unmatched; + }; + rehost + .routes + .decide(value, reference, el.has_attribute("integrity"), page) +} + +/// `value`, a `srcset` or `imagesrcset`, with each candidate's address +/// decided on its own: one a rehosting route serves is written as the +/// route's address or kept as written, and `unmatched` rewrites the others. +/// Every byte between the addresses is kept. +fn rewrite_srcset_candidates( + value: &str, + decide: impl Fn(&str) -> Decision, + unmatched: impl Fn(&str) -> String, +) -> String { + let mut out = String::with_capacity(value.len()); + let mut written = 0; + for range in crate::rehost::srcset_address_ranges(value) { + out.push_str(&value[written..range.start]); + let address = &value[range.clone()]; + match decide(address) { + Decision::Rehost(rehosted) => out.push_str(&rehosted), + Decision::Keep => out.push_str(address), + Decision::Unmatched => out.push_str(&unmatched(address)), + } + written = range.end; + } + out.push_str(&value[written..]); + out +} + /// The processor behind [`create_html_processor`] and /// [`create_html_processor_with_middleware`], carrying out `plan`. /// @@ -680,6 +1042,22 @@ fn build_html_processor( } } + /// Moves only a leading authority (or the supported bare host), never + /// a publisher host occurring inside a path, query or fragment. + fn rewrite_srcset_address(&self, value: &str) -> String { + self.srcset_forms + .iter() + .find_map(|(form, replacement)| { + let rest = value + .get(..form.len())? + .eq_ignore_ascii_case(form) + .then(|| &value[form.len()..])?; + (rest.is_empty() || matches!(rest.as_bytes().first(), Some(b'/' | b'?' | b'#'))) + .then(|| format!("{replacement}{rest}")) + }) + .unwrap_or_else(|| value.to_owned()) + } + fn rewrite_url_value(&self, value: &str) -> Option { if !value.contains(&self.origin_host) { return None; @@ -712,6 +1090,49 @@ fn build_html_processor( &config.request_scheme, )); + // A media or stylesheet address on a rehosting route's third-party folder + // is decided once, on the value the page wrote, before the origin's + // address is moved, so neither rewrite reads the other's output, see + // `crate::rehost`. + struct RehostScope { + rehost: PageRehost, + scheme: String, + host: String, + } + impl RehostScope { + fn decide( + &self, + el: &lol_html::html_content::Element<'_, '_>, + attribute: &str, + original: &str, + ) -> Decision { + if self.rehost.routes.is_empty() { + return Decision::Unmatched; + } + self.decide_decoded(el, attribute, &decode_html_attribute(original)) + } + + fn decide_decoded( + &self, + el: &lol_html::html_content::Element<'_, '_>, + attribute: &str, + value: &str, + ) -> Decision { + let page = PageAddress { + scheme: &self.scheme, + host: &self.host, + removed_prefix: &self.rehost.removed_prefix, + base: self.rehost.base.as_ref(), + }; + rehost_decision(&self.rehost, &page, el, attribute, value) + } + } + let rehost = Rc::new(RehostScope { + rehost: config.rehost.clone(), + scheme: config.request_scheme.clone(), + host: config.request_host.clone(), + }); + let injected_tsjs = Rc::new(Cell::new(false)); let injected_bids = Arc::new(AtomicBool::new(false)); let integration_registry = config.integrations.clone(); @@ -866,11 +1287,20 @@ fn build_html_processor( // Replace URLs in href attributes element!("[href]", { let patterns = patterns.clone(); + let rehost = rehost.clone(); move |el| { - if let Some(href) = el.get_attribute("href") - && let Some(rewritten) = patterns.rewrite_url_value(&href) - { - el.set_attribute("href", &rewritten)?; + if let Some(href) = el.get_attribute("href") { + match rehost.decide(el, "href", &href) { + Decision::Rehost(rehosted) => { + el.set_attribute("href", &url_attribute(rehosted))? + } + Decision::Keep => {} + Decision::Unmatched => { + if let Some(rewritten) = patterns.rewrite_url_value(&href) { + el.set_attribute("href", &rewritten)?; + } + } + } } Ok(()) } @@ -878,11 +1308,32 @@ fn build_html_processor( // Replace URLs in src attributes element!("[src]", { let patterns = patterns.clone(); + let rehost = rehost.clone(); move |el| { - if let Some(src) = el.get_attribute("src") - && let Some(rewritten) = patterns.rewrite_url_value(&src) + if let Some(src) = el.get_attribute("src") { + match rehost.decide(el, "src", &src) { + Decision::Rehost(rehosted) => { + el.set_attribute("src", &url_attribute(rehosted))? + } + Decision::Keep => {} + Decision::Unmatched => { + if let Some(rewritten) = patterns.rewrite_url_value(&src) { + el.set_attribute("src", &rewritten)?; + } + } + } + } + Ok(()) + } + }), + // A video's poster, which only a rehosting route moves. + element!("video[poster]", { + let rehost = rehost.clone(); + move |el| { + if let Some(poster) = el.get_attribute("poster") + && let Decision::Rehost(rehosted) = rehost.decide(el, "poster", &poster) { - el.set_attribute("src", &rewritten)?; + el.set_attribute("poster", &url_attribute(rehosted))?; } Ok(()) } @@ -902,11 +1353,17 @@ fn build_html_processor( // Replace URLs in srcset attributes (for responsive images) element!("[srcset]", { let patterns = patterns.clone(); + let rehost = rehost.clone(); move |el| { if let Some(srcset) = el.get_attribute("srcset") { - let rewritten = replace_each_once(&srcset, &patterns.srcset_forms); - if rewritten != srcset { - el.set_attribute("srcset", &rewritten)?; + let decoded = decode_html_attribute(&srcset); + let rewritten = rewrite_srcset_candidates( + &decoded, + |address| rehost.decide_decoded(&*el, "srcset", address), + |address| patterns.rewrite_srcset_address(address), + ); + if rewritten != decoded.as_ref() { + el.set_attribute("srcset", &url_attribute(rewritten))?; } } Ok(()) @@ -915,11 +1372,17 @@ fn build_html_processor( // Replace URLs in imagesrcset attributes (for link preload) element!("[imagesrcset]", { let patterns = patterns.clone(); + let rehost = rehost.clone(); move |el| { if let Some(imagesrcset) = el.get_attribute("imagesrcset") { - let rewritten = replace_each_once(&imagesrcset, &patterns.url_forms); - if rewritten != imagesrcset { - el.set_attribute("imagesrcset", &rewritten)?; + let decoded = decode_html_attribute(&imagesrcset); + let rewritten = rewrite_srcset_candidates( + &decoded, + |address| rehost.decide_decoded(&*el, "imagesrcset", address), + |address| patterns.rewrite_srcset_address(address), + ); + if rewritten != decoded.as_ref() { + el.set_attribute("imagesrcset", &url_attribute(rewritten))?; } } Ok(()) @@ -927,35 +1390,8 @@ fn build_html_processor( }), ]; - // A response-bound nonce is only safe for the response that carried it, and the - // response-header gate cannot see one the origin delivered in the markup instead. - // Observed structurally rather than by scanning the output bytes, which cannot tell a - // `nonce` attribute from the same word inside a script. if let Some(observed) = config.csp_nonce_observed.clone() { - let meta_observed = Arc::clone(&observed); - element_content_handlers.push(element!("meta[http-equiv][content]", move |el| { - let delivers_csp = el.get_attribute("http-equiv").is_some_and(|equiv| { - matches!( - equiv.trim().to_ascii_lowercase().as_str(), - "content-security-policy" | "content-security-policy-report-only" - ) - }); - if delivers_csp - && el - .get_attribute("content") - .is_some_and(|policy| policy.to_ascii_lowercase().contains("'nonce-")) - { - meta_observed.store(true, Ordering::SeqCst); - } - Ok(()) - })); - // `lol_html` does not entity-decode quoted meta CSP content for the check above. - // Reject nonce attributes independently so an entity-encoded meta policy cannot - // hide executable nonce-bound content from the template-cache safety scan. - element_content_handlers.push(element!("[nonce]", move |_el| { - observed.store(true, Ordering::SeqCst); - Ok(()) - })); + element_content_handlers.extend(markup_nonce_handlers(observed)); } // Shared by every text handler, so each is handed a chunk as the one @@ -1285,7 +1721,9 @@ mod tests { fn create_test_config() -> HtmlProcessorConfig { HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: None, + response_headers: None, public_prefix: None, prepared: Arc::new(Prepared::default()), status: StatusCode::OK, @@ -2038,7 +2476,9 @@ mod tests { #[test] fn injects_ad_slots_at_head_open() { let config = HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: None, + response_headers: None, public_prefix: None, prepared: Arc::new(Prepared::default()), status: StatusCode::OK, @@ -2120,7 +2560,9 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: None, + response_headers: None, public_prefix: None, prepared: Arc::new(Prepared::default()), status: StatusCode::OK, @@ -2163,7 +2605,9 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: None, + response_headers: None, public_prefix: None, prepared: Arc::new(Prepared::default()), status: StatusCode::OK, @@ -2207,7 +2651,9 @@ mod tests { let request_host = "proxy.test-publisher.example.com"; let config = HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: None, + response_headers: None, public_prefix: None, prepared: Arc::new(Prepared::default()), status: StatusCode::OK, @@ -2265,7 +2711,9 @@ mod tests { // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: None, + response_headers: None, public_prefix: None, prepared: Arc::new(Prepared::default()), status: StatusCode::OK, @@ -2301,7 +2749,9 @@ mod tests { // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: None, + response_headers: None, public_prefix: None, prepared: Arc::new(Prepared::default()), status: StatusCode::OK, @@ -2330,7 +2780,9 @@ mod tests { fn marker_mode_config(marker: &str, observer: Option>) -> HtmlProcessorConfig { HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: observer, + response_headers: None, public_prefix: None, body_close: BodyCloseInjection::Marker(marker.to_string()), origin_host: "origin.example.com".to_string(), @@ -2551,7 +3003,9 @@ mod tests { fn bodyless_marker_mode_emits_an_owned_terminal_seam_even_after_source_bytes() { const MARKER: &str = ""; let config = HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: None, + response_headers: None, public_prefix: None, prepared: Arc::new(Prepared::default()), status: StatusCode::OK, @@ -2610,3 +3064,246 @@ mod tests { ); } } + +#[cfg(test)] +mod rehost_tests { + use std::io::Cursor; + + use super::HtmlProcessorConfig; + use super::test_support::{ORIGIN_HOST, REQUEST_HOST, create_page_processor}; + use crate::integrations::IntegrationRegistry; + use crate::middleware::{PhaseEntries, PhaseEntry}; + use crate::settings::{ProxyAssetRoute, Settings}; + use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use crate::test_support::tests::create_test_settings; + + const ROUTE: &str = "/_rehost/cdn.example.net/npm/"; + const PAGE_PREFIX: &str = "/www.publisher.example"; + + fn rehosting_settings() -> Settings { + let mut settings = create_test_settings(); + let mut route = ProxyAssetRoute::new(ROUTE, "https://cdn.example.net/npm"); + route.rehost_page_references = true; + settings.proxy.asset_routes.push(route); + settings.serve = PhaseEntries::new(vec![PhaseEntry { + media_type: "text/css".to_owned(), + path: Some(format!("{ROUTE}moved/")), + middleware: vec!["provenance.encypher.stylesheet".to_owned()], + }]); + settings + } + + fn rendered(settings: &Settings, html: &str) -> String { + rendered_at( + settings, + html, + "https://origin.example.com/articles/page.html?view=1", + ) + } + + fn rendered_at(settings: &Settings, html: &str, document_url: &str) -> String { + let document_url = url::Url::parse(document_url).expect("document URL"); + let base = super::effective_document_base(html.as_bytes(), &document_url) + .expect("should read the original base"); + let registry = IntegrationRegistry::default(); + let config = HtmlProcessorConfig::from_settings( + settings, + ®istry, + ORIGIN_HOST, + REQUEST_HOST, + "https", + ) + .with_removed_prefix(PAGE_PREFIX) + .with_document_base(base); + let mut pipeline = StreamingPipeline::new( + PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size: 64, + }, + create_page_processor(settings, ®istry, config), + ); + let mut output = Vec::new(); + pipeline + .process(Cursor::new(html.as_bytes()), &mut output) + .expect("should process the page"); + String::from_utf8(output).expect("should leave the page UTF-8") + } + + #[test] + fn third_party_media_and_approved_stylesheets_point_at_the_rehosting_route() { + let page = rendered( + &rehosting_settings(), + concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#"a"#, + r#""#, + ), + ); + let reader = format!("https://{REQUEST_HOST}{PAGE_PREFIX}{ROUTE}"); + for expected in [ + format!(r#"href="{reader}moved/a.css?ver=2#x""#), + // A stylesheet no entry moves is kept as written, its query too. + r#"href="//cdn.example.net/npm/kept/b.css?ret=https://origin.example.com/p""# + .to_owned(), + // Integrity is never broken. + r#"href="https://cdn.example.net/npm/moved/c.css?ret=https://origin.example.com/p""# + .to_owned(), + format!(r#"src="{reader}i.png""#), + format!(r#"srcset="{reader}i.png?w=1,2 1x, https://{REQUEST_HOST}/o.png 2x""#), + format!(r#"poster="{reader}p.jpg""#), + // A link a reader follows is not media. + r#"href="https://cdn.example.net/npm/page.html""#.to_owned(), + format!(r#"src="https://{REQUEST_HOST}/own.png""#), + ] { + assert!( + page.contains(&expected), + "should contain {expected}: {page}" + ); + } + } + + #[test] + fn a_page_with_no_rehosting_route_preserves_publisher_names_inside_other_urls() { + let page = rendered( + &create_test_settings(), + r#""#, + ); + assert!( + page.contains(r#"srcset="https://cdn.example.net/x.png?u=origin.example.com 1x""#), + "a publisher host inside another URL's query is not an authority: {page}", + ); + } + + #[test] + fn source_set_fallback_changes_only_the_leading_authority_or_bare_host() { + let value = concat!( + "https://reader.example/www.publisher.example/origin.example.com/p.png?q=origin.example.com#f 1x, ", + "https://third.example/i.png?u=https://origin.example.com/a#origin.example.com 2x, ", + "https://origin.example.com.evil/i.png 3x, ", + "origin.example.com.evil/i.png 4x, ", + "https://origin.example.com/a/origin.example.com.png?q=%2f&x=origin.example.com#f 5x, ", + "//origin.example.com/b.png?x=1,2 6x, ", + "origin.example.com/c.png?q=origin.example.com#f 7x" + ); + let expected = format!( + concat!( + "https://reader.example/www.publisher.example/origin.example.com/p.png?q=origin.example.com#f 1x, ", + "https://third.example/i.png?u=https://origin.example.com/a#origin.example.com 2x, ", + "https://origin.example.com.evil/i.png 3x, ", + "origin.example.com.evil/i.png 4x, ", + "https://{REQUEST_HOST}/a/origin.example.com.png?q=%2f&x=origin.example.com#f 5x, ", + "//{REQUEST_HOST}/b.png?x=1,2 6x, ", + "{REQUEST_HOST}/c.png?q=origin.example.com#f 7x" + ), + REQUEST_HOST = REQUEST_HOST, + ); + for settings in [create_test_settings(), rehosting_settings()] { + let page = rendered( + &settings, + &format!( + r#""# + ), + ); + assert!(page.contains(&format!(r#"srcset="{expected}""#)), "{page}"); + assert!( + page.contains(&format!(r#"imagesrcset="{expected}""#)), + "{page}" + ); + } + } + + #[test] + fn late_first_base_resolves_earlier_references_and_later_bases_do_not_win() { + let page = rendered( + &rehosting_settings(), + concat!( + r#""#, + r#""#, + r#""#, + r#""#, + ), + ); + let reader = format!("https://{REQUEST_HOST}{PAGE_PREFIX}{ROUTE}lib/"); + assert!( + page.contains(&format!(r#"src="{reader}i.png?q=%2f&x=1#f""#)), + "{page}" + ); + assert!( + page.contains(&format!(r#"src="{reader}second.png""#)), + "{page}" + ); + } + + #[test] + fn protocol_relative_references_follow_the_effective_base_scheme() { + let page = rendered( + &rehosting_settings(), + concat!( + r#""#, + r#""#, + ), + ); + assert!( + page.contains(r#"src="//cdn.example.net/npm/a.png""#), + "{page}" + ); + assert!( + !page.contains(&format!("{PAGE_PREFIX}{ROUTE}a.png")), + "{page}" + ); + } + + #[test] + fn base_scan_ignores_comments_rawtext_templates_and_foreign_elements() { + let document = url::Url::parse("https://cdn.example.net/npm/articles/page.html?q=1") + .expect("document URL"); + let source = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + ); + let base = super::effective_document_base(source.as_bytes(), &document).expect("base"); + assert_eq!( + base.as_str(), + "https://cdn.example.net/npm/lib/?x=1&y=%2f#f" + ); + let invalid = super::effective_document_base( + br#""#, + &document, + ) + .expect("invalid first base falls back"); + assert_eq!(invalid, document); + } + + #[test] + fn decoded_query_bytes_are_serialized_once_in_each_rehosted_attribute() { + let value = "https://cdn.example.net/npm/a.png?q=%2f&literal=&amp;#f%2F"; + let page = rendered( + &rehosting_settings(), + &format!( + r#""# + ), + ); + let emitted = format!( + "https://{REQUEST_HOST}{PAGE_PREFIX}{ROUTE}a.png?q=%2f&literal=&amp;#f%2F" + ); + assert!(page.contains(&format!(r#"src="{emitted}""#)), "{page}"); + assert!( + page.contains(&format!(r#"srcset="{emitted} 1x""#)), + "{page}" + ); + assert!( + page.contains(&format!(r#"imagesrcset="{emitted} 2x""#)), + "{page}" + ); + } +} diff --git a/crates/trusted-server-core/src/html_processor/source_protection.rs b/crates/trusted-server-core/src/html_processor/source_protection.rs new file mode 100644 index 0000000000..7e926e0ad7 --- /dev/null +++ b/crates/trusted-server-core/src/html_processor/source_protection.rs @@ -0,0 +1,396 @@ +//! Bounded original-source hold around the entire HTML transformation pass. +//! Classification belongs to the selected middleware, not to core. + +use std::io; +use std::ops::Range; + +use futures::future::LocalBoxFuture; + +use crate::middleware::{MiddlewareChain, SourceProtection}; +use crate::platform::RuntimeServices; +use crate::streaming_processor::StreamProcessor; + +pub(super) struct SourceProtected

{ + inner: P, + classifiers: Vec, + original: Vec, + limit: usize, + ended: bool, +} + +impl

SourceProtected

{ + pub(super) fn new(inner: P, chains: &[&MiddlewareChain], limit: usize) -> Self { + Self { + inner, + classifiers: chains + .iter() + .flat_map(|chain| chain.source_protections()) + .collect(), + original: Vec::new(), + limit, + ended: false, + } + } + + fn hold(&mut self, chunk: &[u8]) -> io::Result<()> { + if self.ended { + return Err(refuse("source protection processor already ended")); + } + append(&mut self.original, chunk, self.limit) + } + + fn mask(&mut self) -> io::Result { + self.ended = true; + let mut ranges: Vec> = Vec::new(); + for classify in &self.classifiers { + let classified = classify(&self.original)?; + let mut previous_end = 0; + for range in &classified { + if range.start < previous_end + || range.start >= range.end + || range.end > self.original.len() + { + return Err(refuse("source protection returned invalid ranges")); + } + previous_end = range.end; + } + ranges.extend(classified); + } + ranges.sort_unstable_by_key(|range| (range.start, range.end)); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + if let Some(last) = merged.last_mut().filter(|last| range.start <= last.end) { + last.end = last.end.max(range.end); + } else { + merged.push(range); + } + } + MaskedDocument::new(std::mem::take(&mut self.original), merged, self.limit) + } +} + +impl StreamProcessor for SourceProtected

{ + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> io::Result> { + if self.classifiers.is_empty() { + return self.inner.process_chunk(chunk, is_last); + } + self.hold(chunk)?; + if !is_last { + return Ok(Vec::new()); + } + let masked = self.mask()?; + let input = masked.input(self.limit)?; + let output = self.inner.process_chunk(&input, true)?; + masked.restore(output, self.limit) + } + + fn finish<'a>( + &'a mut self, + chunk: &'a [u8], + services: &'a RuntimeServices, + ) -> LocalBoxFuture<'a, io::Result>> { + Box::pin(async move { + if self.classifiers.is_empty() { + return self.inner.finish(chunk, services).await; + } + self.hold(chunk)?; + let masked = self.mask()?; + let input = masked.input(self.limit)?; + let output = self.inner.finish(&input, services).await?; + masked.restore(output, self.limit) + }) + } + + fn reset(&mut self) { + self.inner.reset(); + self.original.clear(); + self.ended = false; + } +} + +struct MaskedDocument { + // Retain one original allocation. Protected SVGs remain slices, never copies. + original: Vec, + ranges: Vec>, + prefix: String, +} + +impl MaskedDocument { + fn new(original: Vec, ranges: Vec>, limit: usize) -> io::Result { + if original.len() > limit { + return Err(refuse("source protection original exceeds body limit")); + } + let prefix = if ranges.is_empty() { + String::new() + } else { + format!("", self.prefix) + } + + fn input(&self, limit: usize) -> io::Result> { + if self.ranges.is_empty() { + return Ok(std::borrow::Cow::Borrowed(&self.original)); + } + let mut input = Vec::new(); + let mut cursor = 0; + for (index, range) in self.ranges.iter().enumerate() { + append(&mut input, &self.original[cursor..range.start], limit)?; + append(&mut input, self.marker(index).as_bytes(), limit)?; + cursor = range.end; + } + append(&mut input, &self.original[cursor..], limit)?; + Ok(std::borrow::Cow::Owned(input)) + } + + fn restore(&self, output: Vec, limit: usize) -> io::Result> { + if output.len() > limit { + return Err(refuse("source protection output exceeds body limit")); + } + if self.ranges.is_empty() { + return Ok(output); + } + let mut restored = Vec::new(); + let mut cursor = 0; + for (index, range) in self.ranges.iter().enumerate() { + let at = find(&output[cursor..], self.prefix.as_bytes()) + .map(|at| cursor + at) + .ok_or_else(|| refuse("source protection marker lost"))?; + let marker = self.marker(index); + if !output[at..].starts_with(marker.as_bytes()) { + return Err(refuse( + "source protection marker duplicated, reordered or altered", + )); + } + append(&mut restored, &output[cursor..at], limit)?; + append(&mut restored, &self.original[range.clone()], limit)?; + cursor = at + marker.len(); + } + if find(&output[cursor..], self.prefix.as_bytes()).is_some() { + return Err(refuse("source protection marker duplicated")); + } + append(&mut restored, &output[cursor..], limit)?; + Ok(restored) + } +} + +fn append(output: &mut Vec, bytes: &[u8], limit: usize) -> io::Result<()> { + if bytes.len() > limit.saturating_sub(output.len()) { + return Err(refuse("source protection exceeds body limit")); + } + // Reserve explicitly so Vec's implicit growth cannot overshoot the bound. + let needed = output.len() + bytes.len(); + if needed > output.capacity() { + let capacity = output.capacity().saturating_mul(2).max(needed).min(limit); + output.reserve_exact(capacity - output.len()); + } + output.extend_from_slice(bytes); + Ok(()) +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn refuse(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::middleware::{ + HTML_MEDIA_TYPE, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, + }; + use crate::platform::test_support::noop_services; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct Policy(SourceProtection); + + impl Middleware for Policy { + fn middleware_id(&self) -> &'static str { + "test.protection" + } + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + fn source_protection(&self) -> Option { + Some(self.0) + } + fn create(&self, _: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } + } + + struct Replace { + calls: Arc, + transform: fn(&[u8]) -> Vec, + } + + impl StreamProcessor for Replace { + fn process_chunk(&mut self, chunk: &[u8], _: bool) -> io::Result> { + self.calls.fetch_add(1, Ordering::Relaxed); + Ok((self.transform)(chunk)) + } + } + + fn processor( + classify: SourceProtection, + transform: fn(&[u8]) -> Vec, + limit: usize, + ) -> (SourceProtected, Arc) { + let chain = MiddlewareChain::new( + MiddlewarePhase::Serve, + HTML_MEDIA_TYPE, + vec![Arc::new(Policy(classify))], + ); + let calls = Arc::new(AtomicUsize::new(0)); + ( + SourceProtected::new( + Replace { + calls: Arc::clone(&calls), + transform, + }, + &[&chain], + limit, + ), + calls, + ) + } + + fn middle(_: &[u8]) -> io::Result>> { + Ok(std::iter::once(1..5).collect()) + } + fn unchanged(bytes: &[u8]) -> Vec { + bytes.to_vec() + } + + #[test] + fn holds_original_until_complete_and_restores_exact_bytes_after_async_finish() { + let (mut processor, calls) = processor( + middle, + |bytes| { + let mut output = b"outside".to_vec(); + output.extend_from_slice(&bytes[1..]); + output + }, + 1024, + ); + assert!( + processor + .process_chunk(b"aKEEP", false) + .expect("hold") + .is_empty() + ); + assert_eq!(calls.load(Ordering::Relaxed), 0, "no early transform"); + let result = + futures::executor::block_on(processor.finish(b"z", &noop_services())).expect("restore"); + assert_eq!(result, b"outsideKEEPz"); + assert_eq!(calls.load(Ordering::Relaxed), 1); + } + + #[test] + fn source_limit_is_checked_before_copying_or_running_transforms() { + let (mut processor, calls) = processor(middle, unchanged, 5); + assert!( + processor + .process_chunk(b"a", false) + .expect("hold") + .is_empty() + ); + let before = processor.original.capacity(); + assert!(processor.process_chunk(&[b'x'; 128], false).is_err()); + assert_eq!(processor.original, b"a"); + assert_eq!(processor.original.capacity(), before); + assert_eq!(calls.load(Ordering::Relaxed), 0); + } + + #[test] + fn refuses_invalid_ranges_without_exposing_output() { + for classify in [ + (|_: &[u8]| Ok(std::iter::once(4..7).collect())) as SourceProtection, + |_: &[u8]| Ok(std::iter::once(2..2).collect()), + |_: &[u8]| Ok(vec![1..4, 3..5]), + |_: &[u8]| Ok(vec![3..5, 0..1]), + |_: &[u8]| Err(refuse("classification failed")), + ] { + let (mut processor, calls) = processor(classify, unchanged, 1024); + assert!(processor.process_chunk(b"aKEEPz", true).is_err()); + assert_eq!(calls.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn refuses_lost_duplicated_reordered_and_altered_markers() { + let masked = + MaskedDocument::new(b"aKEEPzOTHERq".to_vec(), vec![1..5, 6..11], 1024).expect("mask"); + let first = masked.marker(0); + let second = masked.marker(1); + for invalid in [ + format!("a{second}q"), + format!("a{first}{first}{second}q"), + format!("a{second}{first}q"), + format!("a{}x-->{second}q", masked.prefix), + format!("a{first}{second}{second}q"), + ] { + assert!(masked.restore(invalid.into_bytes(), 1024).is_err()); + } + assert_eq!( + masked + .restore(format!("edited{first}z{second}q").into_bytes(), 1024) + .expect("restore"), + b"editedKEEPzOTHERq" + ); + } + + #[test] + fn refuses_mask_and_restoration_expansion_past_limit() { + let short = MaskedDocument::new(b"aKEEPz".to_vec(), std::iter::once(1..5).collect(), 6) + .expect("original fits"); + assert!(short.input(6).is_err(), "opaque marker must fit too"); + let original = format!("a{}z", "K".repeat(128)).into_bytes(); + let limit = original.len(); + let masked = MaskedDocument::new( + original.clone(), + std::iter::once(1..limit - 1).collect(), + limit, + ) + .expect("fits"); + let input = masked.input(limit).expect("mask fits").into_owned(); + assert!(masked.restore(input.clone(), limit - 1).is_err()); + assert_eq!(masked.restore(input, limit).expect("exact limit"), original); + } + + #[test] + fn policy_is_default_off_and_selected_only_from_supplied_chains() { + let chain = MiddlewareChain::new(MiddlewarePhase::Serve, HTML_MEDIA_TYPE, Vec::new()); + let calls = Arc::new(AtomicUsize::new(0)); + let mut processor = SourceProtected::new( + Replace { + calls: Arc::clone(&calls), + transform: unchanged, + }, + &[&chain], + 1, + ); + assert_eq!( + processor.process_chunk(b"unheld", false).expect("stream"), + b"unheld" + ); + assert_eq!(calls.load(Ordering::Relaxed), 1); + } +} diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs index 76e9103360..2e9b90a296 100644 --- a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -684,7 +684,9 @@ mod tests { vec![Arc::new(ScriptAddress(integration))], ); let config = HtmlProcessorConfig { + rehost: crate::rehost::PageRehost::default(), csp_nonce_observed: None, + response_headers: None, public_prefix: None, prepared: std::sync::Arc::new(crate::middleware::Prepared::default()), status: http::StatusCode::OK, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index ea672660ff..01b0220613 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -22,6 +22,7 @@ use crate::error::TrustedServerError; use crate::evidence::OwnedRequestInfo; use crate::geo::GeoInfo; use crate::http_util::is_navigation_request; +use crate::media_signing::MediaSigner; use crate::middleware::{Middleware, MiddlewareChain, MiddlewarePhase, PhaseEntries}; use crate::module_context::{ModuleCall, ModuleContext, ModuleRequest, ResolvedRequest}; use crate::origin_auth::OriginSigners; @@ -707,6 +708,11 @@ pub struct IntegrationRegistration { /// on every request, so declaring one runs it wherever the registration /// itself runs. pub identifier_module: Option>, + /// Media signer this registration supplies, see [`crate::media_signing`]. + /// + /// No section selects one. It runs wherever the registration itself + /// runs, and at most one running registration may supply one. + pub media_signer: Option>, } impl IntegrationRegistration { @@ -741,6 +747,7 @@ impl IntegrationRegistrationBuilder { source_module: None, analytics_module: None, identifier_module: None, + media_signer: None, }, } } @@ -872,6 +879,18 @@ impl IntegrationRegistrationBuilder { self } + /// Declare the media signer this registration supplies, which core asks + /// to sign the binary responses it claims, see [`crate::media_signing`]. + /// + /// It runs wherever the registration runs. The registry refuses to build + /// when two running registrations supply one, because signing a body + /// twice would sign the first signer's output as a source. + #[must_use] + pub fn with_media_signer(mut self, signer: Arc) -> Self { + self.registration.media_signer = Some(signer); + self + } + /// Declare the rules this registration adds to `/robots.txt`, under their /// `name`. /// @@ -1025,6 +1044,9 @@ struct IntegrationRegistryInner { // registration order, which core asks on every request. Shared, so a // request's services take a reference count rather than a copy. identifier_modules: Arc<[Arc]>, + // The media signer a running registration supplies, with the + // integration that supplied it. At most one. + media_signer: Option<(&'static str, Arc)>, } /// The sources a deployment's settings name, built once. @@ -1074,6 +1096,7 @@ impl Default for IntegrationRegistryInner { analytics_module: None, sources: None, identifier_modules: Arc::default(), + media_signer: None, } } } @@ -2304,6 +2327,19 @@ impl IntegrationRegistry { )?; identifier_modules.push(module); } + if let Some(signer) = registration.media_signer { + if let Some((supplier, _)) = &inner.media_signer { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integrations `{supplier}` and `{}` both supply a media signer, and \ + at most one running integration may, because a body is signed \ + once. Select only one of their modules", + registration.integration_id + ), + })); + } + inner.media_signer = Some((registration.integration_id, signer)); + } if let Some((name, contributor)) = registration.robots_txt_contributor { claim_module_name( &mut claimed, @@ -2451,6 +2487,17 @@ impl IntegrationRegistry { Arc::clone(&self.inner.identifier_modules) } + /// The media signer the running registrations supply, if one does. + /// Adapters hand it to each request's services with + /// [`RuntimeServices::with_media_signer`]. + #[must_use] + pub fn media_signer(&self) -> Option> { + self.inner + .media_signer + .as_ref() + .map(|(_, signer)| Arc::clone(signer)) + } + /// The prefixes of the publisher `request` was resolved to under /// `settings`, from the selected prefix module. /// @@ -3059,6 +3106,7 @@ impl IntegrationRegistry { analytics_module: None, sources: None, identifier_modules: Arc::default(), + media_signer: None, }), plan: None, platform: Arc::default(), @@ -3147,6 +3195,7 @@ impl IntegrationRegistry { analytics_module: None, sources: None, identifier_modules: Arc::default(), + media_signer: None, }), plan: None, platform: Arc::default(), @@ -4070,6 +4119,35 @@ pub(crate) mod test_support { } } + /// Ends each stylesheet a `[[serve]]` entry names it for with a line, + /// standing in for a stylesheet signer. + pub(crate) const STYLESHEET: &str = "testing.middleware-fixture.stylesheet"; + /// The line [`STYLESHEET`] adds. + pub(crate) const STYLESHEET_LINE: &str = "/* served by the middleware fixture */\n"; + + struct Stylesheet; + + impl Middleware for Stylesheet { + fn middleware_id(&self) -> &'static str { + STYLESHEET + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == "text/css" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + stream: Some(Box::new(AppendLine(STYLESHEET_LINE))), + ..MiddlewareAction::pass() + } + } + } + /// Says it moves the origin's address itself, on its own media type, /// and then leaves the body as it came with a line added, so a test /// sees what core left to it. @@ -4149,6 +4227,7 @@ pub(crate) mod test_support { .with_middleware(Arc::new(Locating)) .with_middleware(Arc::new(Labeled { text: labeled })) .with_middleware(Arc::new(Framed)) + .with_middleware(Arc::new(Stylesheet)) .build(), )) } @@ -4552,6 +4631,7 @@ mod tests { }; use super::*; use crate::constants::COOKIE_EC; + use crate::media_signing::test_support::{Behavior, StubSigner}; use crate::middleware::{HTML_MEDIA_TYPE, MiddlewareAction, MiddlewareContext, PhaseEntry}; use crate::permissions::{Permission, PermissionSet, PermissionState}; use crate::platform::test_support::noop_services; @@ -7687,6 +7767,101 @@ mod tests { ); } + /// The names the two media signer test modules run under. + const MEDIA_PROBE_MODULE: &str = "testing.media-probe"; + const MEDIA_SECOND_MODULE: &str = "testing.media-second"; + + fn media_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("media-probe") + .with_media_signer(StubSigner::new(Behavior::Sign, 1024)) + .build(), + )) + } + + fn media_second_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("media-second") + .with_media_signer(StubSigner::new(Behavior::Sign, 1024)) + .build(), + )) + } + + fn media_probe_builder() -> crate::integrations::IntegrationBuilder { + crate::integrations::IntegrationBuilder::new( + "media-probe", + "seam-probe", + media_probe_registration, + validate_nothing, + ) + .with_module_name(MEDIA_PROBE_MODULE) + } + + /// No section selects a media signer. A running registration's is handed + /// out for each request's services. + #[test] + fn a_running_registration_hands_out_its_media_signer() { + let settings = settings_naming(MEDIA_PROBE_MODULE); + + let registry = IntegrationRegistry::with_registrations(&settings, &[media_probe_builder()]) + .expect("should build a registry with a media signer"); + + assert_eq!( + registry.media_signer().map(|signer| signer.signer_id()), + Some("testing.media"), + "should hand out the running registration's signer" + ); + } + + /// A registration a section does not select runs nothing, its media + /// signer included, so no response is claimed. + #[test] + fn an_unselected_registration_offers_no_media_signer() { + let settings = crate::test_support::tests::create_test_settings(); + + let registry = IntegrationRegistry::with_registrations(&settings, &[media_probe_builder()]) + .expect("should build a registry without the probe selected"); + + assert!( + registry.media_signer().is_none(), + "an unselected registration's signer should not be handed out" + ); + } + + /// Two running signers would sign one signer's output as the other's + /// source, so the pair is refused. + #[test] + fn two_running_registrations_supplying_a_media_signer_are_refused() { + let mut settings = settings_naming(MEDIA_PROBE_MODULE); + settings.select_module("testing", MEDIA_SECOND_MODULE); + let extra = [ + media_probe_builder(), + crate::integrations::IntegrationBuilder::new( + "media-second", + "seam-probe", + media_second_registration, + validate_nothing, + ) + .with_module_name(MEDIA_SECOND_MODULE), + ]; + + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should refuse two media signers"); + + let message = error.to_string(); + assert!( + message.contains("both supply a media signer") + && message.contains("`media-probe`") + && message.contains("`media-second`"), + "error should name both integrations: {message}" + ); + } + fn html_entry(path: Option<&str>, names: &[&str]) -> PhaseEntry { PhaseEntry { media_type: HTML_MEDIA_TYPE.to_owned(), @@ -7712,51 +7887,6 @@ mod tests { let registry = IntegrationRegistry::new(&settings) .expect("should build a registry whose entries name registered middleware"); - assert_eq!( - registry.middleware_ids(), - [ - fixture::HEAD, - fixture::LINKS, - fixture::BROKEN, - fixture::READER, - fixture::BROKEN_READER, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::TEXT_HEADER, - fixture::LOCATING, - fixture::LABELED, - fixture::FRAMED, - crate::service_version::MODULE, - ], - "should list the middleware the selected module supplies, and core's own after them" - ); - assert_eq!( - registry.middleware_in(MiddlewarePhase::Fetch), - [ - fixture::HEAD, - fixture::LINKS, - fixture::BROKEN, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::LABELED, - fixture::FRAMED - ], - "should list the middleware that run in the fetch phase" - ); - assert_eq!( - registry.middleware_in(MiddlewarePhase::Serve), - [ - fixture::READER, - fixture::BROKEN_READER, - fixture::PREPARING, - fixture::TEXT_HEADER, - fixture::LOCATING, - crate::service_version::MODULE - ], - "should list the middleware that run in the serve phase, core's own last" - ); let chain_for = |media_type: &str, path: &str| { registry .middleware_chain(&settings.fetch, MiddlewarePhase::Fetch, media_type, path) @@ -7979,45 +8109,6 @@ mod tests { ); } - #[test] - fn an_entry_naming_a_middleware_nothing_supplies_is_refused_with_what_is_supplied() { - let settings = fixture_settings_with_entries(vec![ - html_entry(Some("/news/"), &[fixture::HEAD]), - html_entry(None, &[fixture::HEAD, "testing.nothing"]), - ]); - - let error = IntegrationRegistry::new(&settings) - .err() - .expect("should refuse an entry naming a middleware nothing supplies"); - - let message = error.to_string(); - assert!( - message.contains( - "[[fetch]] entry 2 names `testing.nothing`, which no module that runs supplies" - ) && message.contains(&format!( - "[{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}]", - fixture::HEAD, - fixture::LINKS, - fixture::BROKEN, - fixture::READER, - fixture::BROKEN_READER, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::TEXT_HEADER, - fixture::LOCATING, - fixture::LABELED, - fixture::FRAMED, - crate::service_version::MODULE - )), - "should name the entry and list what could be named: {message}" - ); - assert!( - !message.contains("no section selects"), - "should not blame a selection when the name is no module's: {message}" - ); - } - #[test] fn an_entry_naming_the_middleware_of_a_module_no_section_selects_says_so() { let mut settings = crate::test_support::tests::create_test_settings(); @@ -8159,59 +8250,6 @@ mod tests { ); } - #[test] - fn a_middleware_no_entry_names_is_listed_for_the_startup_warning() { - let unnamed = |entries: Vec| -> Vec<&'static str> { - let settings = fixture_settings_with_entries(entries); - let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); - unnamed_middleware(&settings, ®istry.inner) - .into_iter() - .map(|(integration, middleware)| { - assert_eq!(integration, fixture::ID, "should name the supplier"); - middleware.middleware_id() - }) - .collect() - }; - - assert_eq!( - unnamed(Vec::new()), - [ - fixture::HEAD, - fixture::LINKS, - fixture::BROKEN, - fixture::READER, - fixture::BROKEN_READER, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::TEXT_HEADER, - fixture::LOCATING, - fixture::LABELED, - fixture::FRAMED, - ], - "should list every middleware when there are no entries" - ); - assert_eq!( - unnamed(vec![ - html_entry(Some("/news/"), &[fixture::LINKS]), - html_entry(None, &[fixture::HEAD]), - ]), - [ - fixture::BROKEN, - fixture::READER, - fixture::BROKEN_READER, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::TEXT_HEADER, - fixture::LOCATING, - fixture::LABELED, - fixture::FRAMED - ], - "should leave out a middleware any entry names" - ); - } - #[test] fn an_entry_naming_a_middleware_in_a_phase_it_does_not_run_in_is_refused() { let mut settings = settings_naming(fixture::MODULE); @@ -8264,26 +8302,6 @@ mod tests { chain_for(MiddlewarePhase::Serve, "/sport/today").is_empty(), "should run no serve middleware on a path only the fetch entries cover" ); - let unnamed: Vec<&str> = unnamed_middleware(&settings, ®istry.inner) - .into_iter() - .map(|(_, middleware)| middleware.middleware_id()) - .collect(); - assert_eq!( - unnamed, - [ - fixture::LINKS, - fixture::BROKEN, - fixture::BROKEN_READER, - fixture::TEXT, - fixture::PREPARING, - fixture::REFUSING, - fixture::TEXT_HEADER, - fixture::LOCATING, - fixture::LABELED, - fixture::FRAMED - ], - "should count a middleware as named only by an entry of a phase it runs in" - ); } /// A middleware under a name of the test's choosing, running in the diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 0422b2882c..f33e9d7361 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -45,6 +45,7 @@ pub mod constants; pub mod cookies; pub mod creative; pub mod creative_opportunities; +pub(crate) mod css_relocation; pub mod ec; pub mod edge_cookie; pub mod error; @@ -56,6 +57,7 @@ pub mod html_processor; pub mod http_util; pub mod inspect; pub mod integrations; +pub mod media_signing; pub mod middleware; pub mod module_context; pub mod module_name; @@ -70,6 +72,7 @@ pub mod provider_table; pub mod proxy; pub mod publisher; pub mod redacted; +pub mod rehost; pub mod request_signing; pub(crate) mod response_header_rewrite; pub mod response_privacy; diff --git a/crates/trusted-server-core/src/media_signing.rs b/crates/trusted-server-core/src/media_signing.rs new file mode 100644 index 0000000000..82c7c7ff4f --- /dev/null +++ b/crates/trusted-server-core/src/media_signing.rs @@ -0,0 +1,2094 @@ +//! Signed media, which a module makes from the exact bytes of a binary +//! response on its way to a reader. +//! +//! A module supplies a [`MediaSigner`] through +//! [`IntegrationRegistrationBuilder::with_media_signer`](crate::integrations::IntegrationRegistrationBuilder::with_media_signer), +//! and it runs wherever the registration runs. Core asks it about two +//! paths: a binary response the publisher path passes through, and the +//! answer of an asset route. Which responses are signed is decided by the +//! origin's media type, so a path with no file extension is covered as an +//! image, audio or video file is. +//! +//! The origin is asked as the reader asked, so a response that is not +//! signed keeps the origin's own range and conditional handling. Where the +//! origin answers `200` with a type the signer handles, within its limit, +//! core captures the body before anything else reads it and hands those +//! exact bytes to the signer. A request that plausibly asks for media asks +//! the origin for an identity-coded body, the only coding signed. Where the +//! origin answers a `206` or a `HEAD` for such a type, core asks it once +//! more for the whole representation, unconditionally, and signs what that +//! answer carries, but only where the signed whole can be kept, so the +//! origin is not read whole for every range. Where the reader's range or +//! `If-Match` brings a `416` or `412` the longer signed representation may +//! answer, and where the reader's `If-Range` or `If-Match` names a signed +//! representation, the origin is asked for the whole representation too. +//! Core applies no transformation of its own, so the captured source is +//! also the final body. +//! +//! An asset route that uses the platform's image optimizer is the one +//! place a transformation happens, and the optimizer reads the origin +//! itself, so core never holds its input. Core asks the optimizer to keep +//! its input's C2PA manifest and add a manifest of its own recording what +//! it did, which binds the input it read by hash. Only a whole `200` to a +//! `GET` whose body carries a C2PA manifest store is handed to the signer, +//! marked as [`MediaJob::optimizer_output`], so the optimizer's own record +//! is the parent of what is signed and core adds nothing to the history. +//! An optimizer writes such a record only where its input carried a +//! manifest, so an unsigned original's optimized answer is never signed. +//! A route may instead send the optimizer to its own signed original on +//! the serving host, see `AssetImageOptimizerConfig::signed_source`, so +//! the optimizer's input is the exact origin bytes core captured and +//! signed. +//! Nothing of it is kept, since its bytes depend on the transformation and +//! the reader's `Accept`, and its signed answer says it varies by `Accept`. +//! +//! The signed bytes are a representation of their own. Core gives them a +//! strong `ETag` of their own digest, a `Repr-Digest` and a +//! `Content-Length`, and answers the reader's `If-Match`, `If-None-Match`, +//! `If-Range` and `Range` against them, with `304`, `412`, `206` and `416` +//! as each requires. The origin's validators never describe signed bytes, +//! so they are removed. Ranges are offered only on a signed representation +//! that was kept. +//! +//! A signed representation is kept in the `[cache.assets]` loading cache +//! for as long as the origin's own headers made it shareably fresh, under +//! the namespace the signer signed it under, and only for a request whose +//! answer the origin could not have made one reader's own. Before the +//! origin is asked, a request of that kind which plausibly asks for media +//! and asks for no revalidation is looked up under the signer's current +//! [`namespace`](MediaSigner::namespace), so a changed or revoked policy +//! never serves a kept copy. A kept copy carries its `Age`, and one whose +//! body no longer hashes to its tag is dropped. +//! +//! What is never signed: a response the origin marked as one reader's, a +//! request carrying `Authorization`, a body larger than the signer's limit, +//! a content-coded body, a body the origin forbade transforming, a +//! response whose own `Link` already names a C2PA manifest, read from the +//! origin's own headers before core writes a cache policy, and a captured +//! body that is an init or media segment of a fragmented ISO BMFF stream, +//! whose provenance binds the stream's parts together. Such a response +//! goes to the reader as the origin answered it, its `Link` untouched. +//! One core meant to +//! sign and could not goes out without the origin's validators and held +//! only briefly and privately, so the next request can be signed. +//! +//! A signer may also sign a source larger than its limit, unchanged, in +//! the large lane, see [`MediaSigner::sign_large`]. Core never holds such a +//! source whole: it pins one version by the origin's strong `ETag` and +//! length, hashes it in bounded chunks, hands the signer a +//! [`VersionedMediaSource`] whose every read is pinned with `If-Match`, and +//! answers the reader with the origin's own bytes and validators plus the +//! signer's headers and a `Repr-Digest`. Only metadata is kept, under the +//! signer's current namespace and the pinned tag. A source without a strong +//! `ETag` goes to the reader as the origin answered it. + +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use base64::Engine as _; +use bytes::{Bytes, BytesMut}; +use edgezero_core::body::Body as EdgeBody; +use futures::StreamExt as _; +use http::{HeaderMap, HeaderName, HeaderValue, Method, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; +use web_time::{SystemTime, UNIX_EPOCH}; + +use crate::middleware::{ResponseHeader, media_type_of}; +use crate::platform::{CacheSpec, LoadError, LoadedValue, RuntimeServices, StoreKey}; +use crate::response_privacy::cache_control_forbids_shared_storage; +use crate::settings::{ASSETS_CACHE, Settings}; + +mod large; + +pub(crate) use large::{LargeAnswer, PinnedOrigin}; +pub use large::{ + LargeMediaJob, LargeMediaSigning, LargeSignedMedia, MediaSourceError, SourceVersion, + VersionedMediaSource, +}; + +/// A module that signs the exact bytes of a binary representation. +/// +/// Registered with +/// [`IntegrationRegistrationBuilder::with_media_signer`](crate::integrations::IntegrationRegistrationBuilder::with_media_signer). +/// At most one running registration may supply one. +pub trait MediaSigner: Send + Sync { + /// The module's name, for logs. + fn signer_id(&self) -> &'static str; + + /// Whether a `GET` or `HEAD` request for `path` may be signed, asked + /// before the origin is. Core changes nothing about the origin request, + /// so this only leaves out paths that never carry media, such as an API. + /// The media type of the origin's answer decides the rest, so a signer + /// answers `true` for a path with no file extension. + fn handles_path(&self, path: &str) -> bool; + + /// Whether this signer signs a body of `media_type`, a bare lower case + /// `type/subtype` read from the origin's `Content-Type`. + fn handles_media_type(&self, media_type: &str) -> bool; + + /// The largest body core captures for this signer. A larger one goes to + /// the reader as the origin answered it, never held whole. + fn max_source_bytes(&self) -> usize; + + /// The policy and credential context a kept signed representation is + /// bound to, from a policy check no older than the signer's own policy + /// lifetime. Core keeps a signed representation under it and serves a + /// kept one only while this answers the same value. An error keeps + /// nothing and serves nothing kept for the request. Never a secret. + fn namespace<'a>(&'a self, services: &'a RuntimeServices) -> MediaNamespacing<'a>; + + /// Signs one captured representation. + fn sign<'a>(&'a self, services: &'a RuntimeServices, job: &'a MediaJob<'a>) + -> MediaSigning<'a>; + + /// The largest source this signer signs unchanged in the large lane, + /// see [`sign_large`](Self::sign_large), or `0`, the default, where it + /// signs none. Core uses the lane only for a source larger than + /// [`max_source_bytes`](Self::max_source_bytes) and no larger than this. + fn max_large_source_bytes(&self) -> u64 { + 0 + } + + /// The one wall-clock budget for a large source: core's hashing of it + /// and the whole of [`sign_large`](Self::sign_large). Core drops the + /// work, stalled reads included, when it is spent. `Duration::ZERO`, + /// the default, keeps the large lane closed, and the lane is also + /// closed on a platform without a timer. + fn max_large_media_duration(&self) -> Duration { + Duration::ZERO + } + + /// Signs one large source that reaches the reader unchanged, from its + /// whole-source digest and pinned reads, returning only the headers its + /// answers carry. + /// + /// Core asks only for a shared `GET` whose origin answer is a `200` or + /// `206` of a type the signer handles, identity coded, not marked as + /// one reader's, not forbidden from transformation, naming no manifest + /// of its own, saying `Accept-Ranges: bytes`, with one strong `ETag` + /// and a known total length above + /// [`max_source_bytes`](Self::max_source_bytes) and within + /// [`max_large_source_bytes`](Self::max_large_source_bytes), on a + /// platform that streams response bodies and offers a timer, and only + /// where no metadata is kept for that version under the current + /// [`namespace`](Self::namespace). The default signs nothing. + fn sign_large<'a>( + &'a self, + _services: &'a RuntimeServices, + _job: &'a LargeMediaJob<'a>, + ) -> LargeMediaSigning<'a> { + Box::pin(async { + Err(MediaSigningFailure { + reason: "large_media_unsupported", + }) + }) + } +} + +/// What [`MediaSigner::sign`] returns. +pub type MediaSigning<'a> = + Pin> + 'a>>; + +/// What [`MediaSigner::namespace`] returns. +pub type MediaNamespacing<'a> = + Pin> + 'a>>; + +/// One captured representation a signer is asked to sign. +#[derive(Debug, Clone, Copy)] +pub struct MediaJob<'a> { + /// The address the reader asked for, `scheme://host/path`, without its + /// query. + pub reader_url: &'a str, + /// The origin's `Content-Type`, as it sent it. + pub content_type: &'a str, + /// The exact body core received, captured before anything else read + /// it. Core transforms nothing, so it is also the final body: the + /// operations core performed are none and the record is complete. + pub source: &'a Bytes, + /// SHA-256 of [`source`](Self::source), lower case hex. + pub source_sha256: &'a str, + /// The address core fetched the source from, exactly as fetched, or + /// `None` when that address carries user information, a query or a + /// fragment, or when the source is + /// [`optimizer_output`](Self::optimizer_output). It names a mutable + /// resource, not an immutable version. + pub source_reference: Option<&'a str>, + /// The lower case host of the address core fetched the source from, + /// with no scheme, port, path, query or user information, present even + /// where [`source_reference`](Self::source_reference) is not. Core + /// always sets it for a body it captured over HTTP. For + /// [`optimizer_output`](Self::optimizer_output) it is the origin host + /// the optimizer was asked to read. + pub source_origin_host: Option<&'a str>, + /// The source is the platform image optimizer's answer, not the + /// origin's bytes. Core hands such a source over only where it carries + /// a C2PA manifest store, which the optimizer writes to record what it + /// did to its own input. A signer must have the signature refused + /// unless that embedded manifest is present and intact, so the + /// optimizer's own record of its input and transformations is the + /// parent of what is signed, never replaced by a claim of core's. + pub optimizer_output: bool, +} + +/// The signed representation a signer returns. +#[derive(Debug, Clone)] +pub struct SignedMedia { + /// The final signed bytes, which core sends as they are. + pub bytes: Bytes, + /// The `Content-Type` of the signed bytes. + pub content_type: HeaderValue, + /// Headers the signer writes on every response of the signed + /// representation: `200`, `206`, `304`, `412`, `416` and `HEAD`. A `Link` + /// is added to the origin's own links, and any other replaces the + /// origin's header of that name. A name core owns, see + /// [`is_reserved_header`], fails the signing. + pub headers: Vec, + /// The policy and credential context these bytes were actually signed + /// under, as [`MediaSigner::namespace`] gives it for that context. Core + /// keeps the signed representation under this value, never under a + /// namespace asked for again, so bytes signed under one policy are never + /// served while another is current. + pub namespace: String, +} + +/// Why a signer did not sign, which core logs before it sends the origin's +/// bytes unsigned. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +#[display("{reason}")] +pub struct MediaSigningFailure { + /// A short code naming the reason. + pub reason: &'static str, +} + +impl core::error::Error for MediaSigningFailure {} + +/// Headers core writes on a signed representation, which a signer may not +/// set. +const RESERVED_HEADERS: [&str; 17] = [ + "content-length", + "content-type", + "content-encoding", + "content-range", + "accept-ranges", + "etag", + "last-modified", + "cache-control", + "expires", + "vary", + "set-cookie", + "transfer-encoding", + "repr-digest", + "content-digest", + "digest", + "age", + "date", +]; + +/// Origin headers that describe the origin's bytes or one transfer of them, +/// never signed bytes. +const ORIGIN_REPRESENTATION_HEADERS: [&str; 11] = [ + "content-length", + "content-encoding", + "content-range", + "accept-ranges", + "etag", + "last-modified", + "repr-digest", + "content-digest", + "digest", + "content-md5", + "transfer-encoding", +]; + +/// Origin headers a kept signed representation carries besides the ones +/// core and the signer write. Core's per-request headers are written again +/// on each reader's copy, so none of them is kept. +const KEPT_ORIGIN_HEADERS: [&str; 6] = [ + "cache-control", + "expires", + "content-disposition", + "content-language", + "link", + "x-content-type-options", +]; + +/// Request headers that make an origin answer with less than the whole +/// representation. +const PRECONDITION_HEADERS: [HeaderName; 6] = [ + header::RANGE, + header::IF_RANGE, + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::IF_MATCH, + header::IF_UNMODIFIED_SINCE, +]; + +/// The first bytes of a kept signed representation, naming its layout. +const KEPT_MAGIC: &[u8; 4] = b"TSM2"; + +/// How many kept signed representations one process holds in memory. Each +/// may be as large as a signer's limit, so few are held. +const KEPT_MEMORY_ENTRIES: usize = 4; + +/// What starts the opaque part of a signed representation's entity tag, +/// so a reader's precondition naming signed bytes is told from one naming +/// the origin's. +const SIGNED_TAG_PREFIX: &str = "sig-"; + +/// The `Cache-Control` an answer core meant to sign and could not goes out +/// with, so no cache holds unsigned bytes in place of signed ones for long. +const UNSIGNED_FALLBACK_CACHE_CONTROL: &str = "private, max-age=60"; + +/// `Sec-Fetch-Dest` values a browser sends for media. +const MEDIA_DESTINATIONS: [&str; 7] = [ + "image", "audio", "video", "track", "font", "object", "embed", +]; + +/// File extensions of media, for a request that names neither a media +/// destination nor a media type it accepts. +const MEDIA_EXTENSIONS: [&str; 33] = [ + "jpg", "jpeg", "png", "gif", "webp", "avif", "heic", "heif", "tif", "tiff", "svg", "bmp", + "ico", "jxl", "mp4", "m4v", "mov", "m4a", "mp3", "wav", "aac", "flac", "ogg", "oga", "ogv", + "opus", "webm", "avi", "mkv", "woff", "woff2", "ttf", "otf", +]; + +/// What a lookup that finds nothing kept answers, never stored. +const NOT_KEPT: &str = "no signed representation kept"; + +/// Whether `name` is a header core owns on a signed representation. +#[must_use] +pub fn is_reserved_header(name: &HeaderName) -> bool { + RESERVED_HEADERS.contains(&name.as_str()) +} + +/// Removes every header that would make the origin answer with less than +/// the whole representation, for the second, whole fetch of a claim. +pub(crate) fn strip_representation_preconditions(headers: &mut HeaderMap) { + for name in &PRECONDITION_HEADERS { + headers.remove(name); + } +} + +/// What the reader's request asked of the representation. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct ReaderPreconditions { + if_match: Option, + if_none_match: Option, + if_range: Option, + range: Option, + if_modified_since: Option, + if_unmodified_since: Option, +} + +impl ReaderPreconditions { + fn of(headers: &HeaderMap) -> Self { + Self { + if_match: joined(headers, &header::IF_MATCH), + if_none_match: joined(headers, &header::IF_NONE_MATCH), + if_range: joined(headers, &header::IF_RANGE), + range: joined(headers, &header::RANGE), + if_modified_since: joined(headers, &header::IF_MODIFIED_SINCE), + if_unmodified_since: joined(headers, &header::IF_UNMODIFIED_SINCE), + } + } + + /// Whether `If-Range` or `If-Match` names a signed representation's + /// entity tag, see [`SIGNED_TAG_PREFIX`]. + fn names_signed_bytes(&self) -> bool { + let if_range = self + .if_range + .as_deref() + .and_then(single_entity_tag) + .is_some_and(|tag| is_signed_tag(&tag)); + let if_match = self + .if_match + .as_deref() + .is_some_and(|value| match TagList::parse(value) { + TagList::Any => false, + TagList::Tags(tags) => tags.iter().any(is_signed_tag), + }); + if_range || if_match + } +} + +/// Whether `tag` is a strong entity tag core gave a signed representation. +fn is_signed_tag(tag: &EntityTag<'_>) -> bool { + !tag.weak + && tag + .opaque + .strip_prefix(SIGNED_TAG_PREFIX) + .is_some_and(|digest| { + digest.len() == 64 + && digest + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) + }) +} + +/// The strong entity tag of signed bytes whose SHA-256 is `digest`. +fn signed_entity_tag(digest: &[u8]) -> String { + format!("\"{SIGNED_TAG_PREFIX}{}\"", hex::encode(digest)) +} + +/// Whether a request plausibly asks for media: the destination a browser +/// names, else the first type it accepts, else the path's file extension. +fn request_looks_like_media(path: &str, headers: &HeaderMap) -> bool { + if let Some(destination) = headers + .get("sec-fetch-dest") + .and_then(|value| value.to_str().ok()) + { + let destination = destination.trim(); + return MEDIA_DESTINATIONS + .iter() + .any(|media| destination.eq_ignore_ascii_case(media)); + } + let accepted = header_text(headers, &header::ACCEPT) + .split(',') + .next() + .map(media_type_of) + .unwrap_or_default(); + if ["image/", "audio/", "video/", "font/"] + .iter() + .any(|prefix| accepted.starts_with(prefix)) + { + return true; + } + path.rsplit('/') + .next() + .and_then(|name| name.rsplit_once('.')) + .is_some_and(|(_, extension)| { + MEDIA_EXTENSIONS + .iter() + .any(|media| extension.eq_ignore_ascii_case(media)) + }) +} + +/// Seconds since the Unix epoch now, from the clock every runtime offers. +fn unix_seconds() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_secs()) +} + +/// An answer core meant to sign and could not, made safe to send unsigned: +/// without the origin's validators, which a later conditional request would +/// use to pin the unsigned bytes, and held only briefly and only by the +/// reader, so the next request can be signed. +fn unsigned_fallback(mut response: Response) -> Response { + crate::cache_policy::remove_owned_cache_headers(&mut response); + let headers = response.headers_mut(); + for name in ["etag", "last-modified", "expires"] { + headers.remove(name); + } + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_static(UNSIGNED_FALLBACK_CACHE_CONTROL), + ); + response +} + +/// Every line of header `name`, joined as one list. A value that is not +/// text stays present and matches nothing. +fn joined(headers: &HeaderMap, name: &HeaderName) -> Option { + let mut values = headers.get_all(name).iter().peekable(); + values.peek()?; + Some( + values + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + .collect::>() + .join(", "), + ) +} + +/// What names one signed representation before the origin is asked: the +/// reader's address, the origin core asks and the address it asks there. +#[derive(Debug, Clone, Copy)] +pub(crate) struct VariantIdentity<'a> { + /// `scheme://host/path` the reader asked for, without its query. + pub reader_url: &'a str, + /// What tells this origin apart from another at one address. + pub origin_identity: &'a str, + /// The address core fetches, exactly as fetched. + pub fetched_url: &'a str, + /// The reader's request carried no query, so + /// [`reader_url`](Self::reader_url) is the whole address it asked for. + pub reader_query_free: bool, +} + +/// Independent remaining browser and shared-cache lifetimes. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct OriginFreshness { + pub browser: Duration, + pub shared: Duration, +} + +/// What the origin's own headers said about its answer, read before core +/// changed or removed any of them, so a cache policy core writes never +/// stands in for the origin's. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct OriginFacts { + /// The origin marked the answer as one reader's, by a cookie, a + /// `Cache-Control` that forbids shared storage or a `Vary` on the + /// reader. + private: bool, + /// The origin forbade transforming the body. + no_transform: bool, + /// How long a signed representation of the answer may be kept. + keep_for: Option, + /// Freshness at receipt, before any response processing. + freshness: OriginFreshness, +} + +impl OriginFacts { + /// What `headers`, the origin's as it sent them, say. + pub(crate) fn of(headers: &HeaderMap) -> Self { + Self::of_at(headers, SystemTime::now()) + } + + /// Read against an explicit receipt time, also used by deterministic tests. + pub(crate) fn of_at(headers: &HeaderMap, received_at: SystemTime) -> Self { + let freshness = origin_freshness(headers, received_at); + Self { + private: headers.contains_key(header::SET_COOKIE) + || cache_control_forbids_shared_storage(headers) + || varies_by_reader(headers), + no_transform: cache_control_has(headers, "no-transform"), + keep_for: (!freshness.shared.is_zero()).then_some(freshness.shared), + freshness, + } + } + + /// The origin marked the answer as one reader's, by a cookie, a + /// `Cache-Control` that forbids shared storage or a `Vary` on the + /// reader. + pub(crate) fn is_private(&self) -> bool { + self.private + } + + /// The origin forbade transforming the body, by `no-transform`. + pub(crate) fn forbids_transform(&self) -> bool { + self.no_transform + } + + /// Freshness after processing, without borrowing the shared lifetime for + /// the browser or granting a new lifetime when origin dates are removed. + pub(crate) fn remaining_freshness(&self, elapsed: Duration) -> OriginFreshness { + OriginFreshness { + browser: self.freshness.browser.saturating_sub(elapsed), + shared: self.freshness.shared.saturating_sub(elapsed), + } + } +} + +/// Where the response handed to [`MediaClaim::deliver`] came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Answered { + /// The origin, to the request as core sent it, with what its own + /// headers said. + Origin(OriginFacts), + /// A kept signed representation [`MediaClaim::kept`] found. + Kept, +} + +/// What [`MediaClaim::deliver`] decided. +pub(crate) enum Delivery { + /// The response to send. + Send(Response), + /// The origin answered with a part, a `HEAD`, or a `412` or `416` a + /// reader's signed-length range or signed tag brings, for a + /// representation that may be signed. The caller asks the origin once + /// more for the whole representation, see + /// [`strip_representation_preconditions`], and hands the answer to + /// [`MediaClaim::deliver_whole`], with this response to send where that + /// fails. + FetchWhole(Response), + /// The origin answered with a source larger than the signer's bounded + /// limit that the large lane admits, pinned to its version. The caller + /// hands it to [`MediaClaim::deliver_large`] with how the origin is + /// asked again. + Large(LargeAnswer), +} + +/// A request the services' [`MediaSigner`] may sign the answer to. +pub(crate) struct MediaClaim { + signer: Arc, + reader_method: Method, + preconditions: ReaderPreconditions, + spec: CacheSpec, + /// The answer may be kept for, and a kept one served to, other + /// readers: the request carries nothing that could make the origin's + /// answer this reader's own. + shared: bool, + /// The request lets a kept signed representation answer it: it is + /// shared, does not ask for revalidation, and plausibly asks for media. + reads_kept: bool, + /// The request plausibly asks for media. + looks_like_media: bool, + /// The reader's `If-Range` or `If-Match` names a signed + /// representation, which the origin cannot judge. + names_signed_bytes: bool, + /// The answer is the platform image optimizer's, see + /// [`answered_by_optimizer`](Self::answered_by_optimizer). + optimizer_output: bool, + /// The claim answered the reader's preconditions against the + /// representation it delivered, see + /// [`answer_unsigned`](Self::answer_unsigned). + conditions_answered: AtomicBool, +} + +impl MediaClaim { + /// The claim the request's services' signer makes on a request, or + /// `None` where no signer is registered, the method is neither `GET` + /// nor `HEAD`, the request carries `Authorization`, or the signer leaves + /// `path` alone. + /// + /// `shared` says whether nothing the request carries beyond these + /// headers could make the origin's answer this reader's own, being a + /// cookie sent to the origin, a module's request state or a + /// personalized delivery. Only a shared request reads or writes a kept + /// signed representation. + pub(crate) fn of_request( + settings: &Settings, + services: &RuntimeServices, + method: &Method, + path: &str, + headers: &HeaderMap, + shared: bool, + ) -> Option { + let signer = services.media_signer()?; + let claimed = (*method == Method::GET || *method == Method::HEAD) + && !headers.contains_key(header::AUTHORIZATION) + && signer.handles_path(path); + if !claimed { + return None; + } + let preconditions = ReaderPreconditions::of(headers); + let looks_like_media = request_looks_like_media(path, headers); + let names_signed_bytes = preconditions.names_signed_bytes(); + Some(Self { + signer: Arc::clone(signer), + reader_method: method.clone(), + preconditions, + spec: CacheSpec { + name: ASSETS_CACHE, + lifetimes: settings + .cache + .assets + .lifetimes(crate::publisher::ASSET_DEFAULT_LIFETIMES), + memory_entries: KEPT_MEMORY_ENTRIES, + }, + shared, + reads_kept: shared + && looks_like_media + && !crate::publisher::request_asks_for_revalidation(headers), + looks_like_media, + names_signed_bytes, + optimizer_output: false, + conditions_answered: AtomicBool::new(false), + }) + } + + /// The same claim on a request the platform's image optimizer answers. + /// + /// The optimizer reads the origin itself, so core never holds its + /// input. Its answer is signed only where it carries a C2PA manifest + /// store, the optimizer's own record of the input it read and what it + /// did, and the signer must refuse unless that manifest is present and + /// intact, see [`MediaJob::optimizer_output`]. Its answer depends on + /// the transformation and the reader's `Accept`, neither of which names + /// a kept representation, so nothing is read from or written to the + /// kept representations, and only a whole `200` to a `GET` is signed: + /// a part or `HEAD` is never asked for again past the optimizer. + pub(crate) fn answered_by_optimizer(mut self) -> Self { + self.shared = false; + self.reads_kept = false; + self.optimizer_output = true; + self + } + + /// Whether the request plausibly asks for media, by the destination + /// the browser names, the type it accepts first, or the file extension. + /// The origin is then asked for an identity-coded body, the only coding + /// signed. + pub(crate) fn looks_like_media(&self) -> bool { + self.looks_like_media + } + + /// Whether the reader's `If-Range` or `If-Match` names a signed + /// representation. The origin cannot judge such a precondition, so the + /// caller asks it for the whole representation, see + /// [`strip_representation_preconditions`], and core answers the + /// reader's range and preconditions against the signed bytes. + pub(crate) fn names_signed_bytes(&self) -> bool { + self.names_signed_bytes + } + + /// Whether the answer is the platform image optimizer's, see + /// [`answered_by_optimizer`](Self::answered_by_optimizer). + pub(crate) fn signs_optimizer_output(&self) -> bool { + self.optimizer_output + } + + /// The signed representation kept for `identity` under the signer's + /// current namespace, as an origin answer to hand to + /// [`deliver`](Self::deliver) with [`Answered::Kept`], or `None`. Only + /// a shared request that plausibly asks for media and asks for no + /// revalidation is looked up, so the signer's namespace is not resolved + /// for any other. + pub(crate) async fn kept( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + ) -> Option> { + if !self.reads_kept || !services.loading_cache_for(ASSETS_CACHE).keeps_values() { + return None; + } + let namespace = match self.signer.namespace(services).await { + Ok(namespace) => namespace, + Err(failure) => { + log::debug!( + "media signing: nothing kept looked up signer={} reason={failure}", + self.signer.signer_id() + ); + return None; + } + }; + let key = self.store_key(services, &namespace, identity); + let bytes = services + .load_cached( + &self.spec, + &key, + Box::new(|| Box::pin(async { Err(LoadError::new(NOT_KEPT)) })), + ) + .await + .ok()?; + let kept = decode_kept(&bytes, unix_seconds()); + if kept.is_none() { + log::warn!( + "media signing: kept representation unreadable or altered signer={}", + self.signer.signer_id() + ); + services + .loading_cache_for(ASSETS_CACHE) + .remove(&services.clone().without_connection(), &self.spec, &key) + .await; + } + kept + } + + /// Answers the reader from `response`: a kept signed representation, or + /// the origin's answer, signed where the signer signs it. + pub(crate) async fn deliver( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + answered: Answered, + response: Response, + ) -> Delivery { + let facts = match answered { + Answered::Kept => return Delivery::Send(self.respond_kept(response)), + Answered::Origin(facts) => facts, + }; + if self.optimizer_output { + return Delivery::Send( + self.deliver_optimized(services, identity, &facts, response) + .await, + ); + } + let signer_id = self.signer.signer_id(); + let status = response.status(); + let limit = to_u64(self.signer.max_source_bytes()); + match status { + StatusCode::OK if self.reader_method != Method::HEAD => { + if let Some(reason) = self.unsignable_reason(response.headers(), &facts) { + log::debug!( + "media signing: outcome=unsigned signer={signer_id} reason={reason}" + ); + return Delivery::Send(response); + } + let total = declared_length(response.headers()); + if total.is_some_and(|total| total > limit) { + return self.large_or_unsigned(services, response, total, &facts); + } + let (parts, body) = response.into_parts(); + let source = match self.capture_within_limit(&parts.headers, body).await { + Ok(source) => source, + Err(body) => return Delivery::Send(Response::from_parts(parts, body)), + }; + if is_fragmented_stream_part(&source) { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=fragmented_stream_part" + ); + return Delivery::Send(Response::from_parts(parts, EdgeBody::Once(source))); + } + match self.sign(services, identity, &parts.headers, &source).await { + Ok(signed) => Delivery::Send( + self.respond_signed(services, identity, parts, signed, facts.keep_for) + .await, + ), + Err(()) => Delivery::Send(unsigned_fallback(Response::from_parts( + parts, + EdgeBody::Once(source), + ))), + } + } + StatusCode::OK | StatusCode::PARTIAL_CONTENT => { + if let Some(reason) = self.unsignable_reason(response.headers(), &facts) { + log::debug!( + "media signing: outcome=unsigned signer={signer_id} reason={reason}" + ); + return Delivery::Send(response); + } + let total = if status == StatusCode::PARTIAL_CONTENT { + content_range_total(response.headers()) + } else { + declared_length(response.headers()) + }; + let too_large = if status == StatusCode::PARTIAL_CONTENT { + total.is_none_or(|total| total > limit) + } else { + total.is_some_and(|total| total > limit) + }; + if too_large { + return self.large_or_unsigned(services, response, total, &facts); + } + // A part or a `HEAD` is signed only where the signed whole + // can be kept, so the reader's next range is answered from + // it and the origin is not read whole for every range. + if !self.retains(services, &facts) { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=part_not_kept" + ); + return Delivery::Send(response); + } + Delivery::FetchWhole(response) + } + StatusCode::PRECONDITION_FAILED | StatusCode::RANGE_NOT_SATISFIABLE + if self.preconditions.range.is_some() || self.preconditions.if_match.is_some() => + { + // The reader's range or `If-Match` may describe the signed + // representation, which is longer than the origin's and has + // a tag of its own, so the origin's refusal is not the + // answer until the signed bytes are asked. + let too_large = + content_range_total(response.headers()).is_some_and(|total| total > limit); + if facts.private || too_large { + return Delivery::Send(response); + } + Delivery::FetchWhole(response) + } + StatusCode::NOT_MODIFIED => Delivery::Send( + self.decorate_not_modified(services, identity, &facts, response) + .await, + ), + _ => Delivery::Send(response), + } + } + + /// Answers the reader from the platform image optimizer's answer: a + /// whole `200` to a `GET` that carries a C2PA manifest store is signed, + /// and every other answer goes as the optimizer made it. + async fn deliver_optimized( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + facts: &OriginFacts, + response: Response, + ) -> Response { + let signer_id = self.signer.signer_id(); + if response.status() != StatusCode::OK || self.reader_method != Method::GET { + return response; + } + if let Some(reason) = self.unsignable_reason(response.headers(), facts) { + log::debug!("media signing: outcome=unsigned signer={signer_id} reason={reason}"); + return response; + } + let (parts, body) = response.into_parts(); + let source = match self.capture_within_limit(&parts.headers, body).await { + Ok(source) => source, + Err(body) => return Response::from_parts(parts, body), + }; + // The optimizer writes a manifest only where its input carried one. + // Without it nothing records what the optimizer read or did, so the + // answer is not signed. + if !carries_c2pa_manifest_store(&source) { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=image_optimizer_output_without_c2pa" + ); + return Response::from_parts(parts, EdgeBody::Once(source)); + } + // The optimizer may choose its output format from the reader's + // `Accept`, and it removes the origin's `Vary`, so a cache past core + // is told the signed bytes depend on it. + let mut parts = parts; + let varies_by_accept = parts + .headers + .get_all(header::VARY) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .any(|name| name.trim() == "*" || name.trim().eq_ignore_ascii_case("accept")); + if !varies_by_accept { + parts + .headers + .append(header::VARY, HeaderValue::from_static("Accept")); + } + match self.sign(services, identity, &parts.headers, &source).await { + Ok(signed) => { + self.respond_signed(services, identity, parts, signed, None) + .await + } + Err(()) => unsigned_fallback(Response::from_parts(parts, EdgeBody::Once(source))), + } + } + + /// Answers the reader from the origin's answer to the whole, + /// unconditional request [`Delivery::FetchWhole`] asked for, or sends + /// `fallback` where it cannot be signed. + /// + /// `whole` is the origin's answer as it came, whose own headers decide + /// what is signed and how long it is kept. The signed response is + /// written on `fallback`'s headers, which carry what core wrote for + /// this reader. + pub(crate) async fn deliver_whole( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + whole: Response, + fallback: Response, + ) -> Response { + let signer_id = self.signer.signer_id(); + // The answer sent where the whole cannot be signed already declares + // a manifest of the origin's own, which core never competes with. + if carries_manifest_link(fallback.headers()) { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=origin_manifest_link" + ); + return fallback; + } + let facts = OriginFacts::of(whole.headers()); + if whole.status() != StatusCode::OK { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=whole_fetch_status_{}", + whole.status().as_u16() + ); + return unsigned_fallback(fallback); + } + if let Some(reason) = self.unsignable_reason(whole.headers(), &facts) { + log::info!("media signing: outcome=unsigned signer={signer_id} reason={reason}"); + return fallback; + } + let (whole_parts, whole_body) = whole.into_parts(); + let Ok(source) = self + .capture_within_limit(&whole_parts.headers, whole_body) + .await + else { + return fallback; + }; + // A part of a fragmented stream goes as the origin answered it. + if is_fragmented_stream_part(&source) { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=fragmented_stream_part" + ); + return fallback; + } + match self + .sign(services, identity, &whole_parts.headers, &source) + .await + { + Ok(signed) => { + let (parts, _) = fallback.into_parts(); + self.respond_signed(services, identity, parts, signed, facts.keep_for) + .await + } + Err(()) => unsigned_fallback(fallback), + } + } + + /// Whether a signed whole made for this request can be kept: the request + /// is shared, the origin gave its answer a shared freshness, and the + /// deployment's asset cache keeps values. + fn retains(&self, services: &RuntimeServices, facts: &OriginFacts) -> bool { + self.shared + && facts.keep_for.is_some() + && services.loading_cache_for(ASSETS_CACHE).keeps_values() + } + + /// The whole body of a signable `200`, or the body as it stands where + /// it is larger than the signer's limit or could not be read, so the + /// reader still receives every byte the origin sent. + async fn capture_within_limit( + &self, + headers: &HeaderMap, + body: EdgeBody, + ) -> Result { + let signer_id = self.signer.signer_id(); + let limit = self.signer.max_source_bytes(); + if declared_length(headers).is_some_and(|length| length > to_u64(limit)) { + log::info!("media signing: outcome=unsigned signer={signer_id} reason=exceeds_limit"); + return Err(body); + } + match capture(body, limit).await { + Captured::Whole(source) => Ok(source), + Captured::Partial(body) => { + log::info!( + "media signing: outcome=unsigned signer={signer_id} reason=exceeds_limit_or_unreadable" + ); + Err(body) + } + } + } + + /// Why a response is not handed to the signer, or `None` when it is. + /// `facts` are the origin's own, read before core wrote its cache + /// policy on `headers`. + fn unsignable_reason(&self, headers: &HeaderMap, facts: &OriginFacts) -> Option<&'static str> { + if facts.private || headers.contains_key(header::SET_COOKIE) { + return Some("origin_private"); + } + // Provenance the origin already links to describes these bytes, so + // they are neither signed again nor given a competing manifest. + if carries_manifest_link(headers) { + return Some("origin_manifest_link"); + } + if facts.no_transform { + return Some("no_transform"); + } + let encoding = header_text(headers, &header::CONTENT_ENCODING).trim(); + if !encoding.is_empty() && !encoding.eq_ignore_ascii_case("identity") { + return Some("content_encoded"); + } + let media_type = media_type_of(header_text(headers, &header::CONTENT_TYPE)); + if media_type.is_empty() || !self.signer.handles_media_type(&media_type) { + return Some("media_type"); + } + None + } + + /// Asks the signer for the signed representation of `source`, logging + /// the outcome with no address, since a reader's or origin's address + /// may carry a credential. + async fn sign( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + headers: &HeaderMap, + source: &Bytes, + ) -> Result { + let signer_id = self.signer.signer_id(); + let source_sha256 = hex::encode(Sha256::digest(source)); + let origin_host = url::Url::parse(identity.fetched_url) + .ok() + .and_then(|url| url.host_str().map(str::to_ascii_lowercase)); + let job = MediaJob { + reader_url: identity.reader_url, + content_type: header_text(headers, &header::CONTENT_TYPE), + source, + source_sha256: &source_sha256, + // No address serves the optimizer's answer. + source_reference: if self.optimizer_output { + None + } else { + source_reference(identity.fetched_url) + }, + source_origin_host: origin_host.as_deref(), + optimizer_output: self.optimizer_output, + }; + let checked = match self.signer.sign(services, &job).await { + Ok(signed) if signed.bytes.is_empty() => Err(MediaSigningFailure { + reason: "empty_signed_body", + }), + Ok(signed) + if signed + .headers + .iter() + .any(|(name, _)| is_reserved_header(name)) => + { + Err(MediaSigningFailure { + reason: "reserved_header", + }) + } + Ok(signed) if signed.namespace.is_empty() => Err(MediaSigningFailure { + reason: "no_namespace", + }), + other => other, + }; + match checked { + Ok(signed) => { + log::info!( + "media signing: outcome=signed signer={signer_id} source_bytes={} signed_bytes={}", + source.len(), + signed.bytes.len() + ); + Ok(signed) + } + Err(failure) => { + log::warn!("media signing: outcome=unsigned signer={signer_id} reason={failure}"); + Err(()) + } + } + } + + /// The signed representation, kept for `keep_for` where this request is + /// shared, and answered against the reader's request. Ranges are offered + /// only on a representation that was kept, so a reader's later range is + /// answered from it and never from a part of the origin's own bytes. + async fn respond_signed( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + mut parts: http::response::Parts, + signed: SignedMedia, + keep_for: Option, + ) -> Response { + let origin_age = digits(header_text(&parts.headers, &header::AGE).trim()).unwrap_or(0); + for name in ORIGIN_REPRESENTATION_HEADERS { + parts.headers.remove(name); + } + let digest = Sha256::digest(&signed.bytes); + let etag = signed_entity_tag(digest.as_slice()); + let repr_digest = format!( + "sha-256=:{}:", + base64::engine::general_purpose::STANDARD.encode(digest.as_slice()) + ); + let mut representation = HeaderMap::new(); + representation.insert(header::CONTENT_TYPE, signed.content_type); + representation.insert( + header::ETAG, + HeaderValue::from_str(&etag).expect("should build a hex entity tag"), + ); + representation.insert( + HeaderName::from_static("repr-digest"), + HeaderValue::from_str(&repr_digest).expect("should build a base64 digest"), + ); + for name in KEPT_ORIGIN_HEADERS { + for value in parts.headers.get_all(name) { + representation.append(HeaderName::from_static(name), value.clone()); + } + } + for (name, value) in signed.headers { + if name == header::LINK { + representation.append(name, value); + } else { + representation.insert(name, value); + } + } + let kept = match keep_for.filter(|_| self.shared) { + Some(lifetime) => { + let value = encode_kept(&KeptRecord { + kept_at: unix_seconds(), + age: origin_age, + headers: &representation, + body: &signed.bytes, + }); + self.keep(services, identity, &signed.namespace, value, lifetime) + .await + } + None => false, + }; + for name in representation.keys() { + parts.headers.remove(name); + } + for (name, value) in &representation { + parts.headers.append(name.clone(), value.clone()); + } + self.answer(parts, signed.bytes, kept) + } + + /// A kept signed representation, answered against the reader's + /// request. + fn respond_kept(&self, response: Response) -> Response { + let (parts, body) = response.into_parts(); + let bytes = body.into_bytes().unwrap_or_default(); + self.answer(parts, bytes, true) + } + + /// Answers the reader's preconditions and range against the signed + /// `bytes`, whose `ETag` is in `parts`. + fn answer( + &self, + parts: http::response::Parts, + bytes: Bytes, + ranges_offered: bool, + ) -> Response { + let length = to_u64(bytes.len()); + let etag = parts + .headers + .get(header::ETAG) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let outcome = evaluate( + &self.reader_method, + &self.preconditions, + etag.as_deref(), + None, + length, + ); + self.conditions_answered.store(true, Ordering::Relaxed); + self.apply(parts, bytes, outcome, ranges_offered) + } + + /// `response`, a whole `200` this claim delivered, answered against the + /// reader's preconditions where the claim has not already answered them, + /// which it has for every signed representation it delivered. For a + /// request whose preconditions were not sent to the origin, so the + /// origin's whole answer is checked against them here, by the + /// representation's own `ETag` and `Last-Modified`. A range is not + /// applied, so a passing answer goes whole, as RFC 9110 allows. + pub(crate) fn answer_unsigned(&self, response: Response) -> Response { + if self.conditions_answered.load(Ordering::Relaxed) { + return response; + } + answer_whole_with(&self.reader_method, &self.preconditions, response) + } +} + +/// `response`, a whole `200`, answered against the preconditions of a +/// reader's request carrying `reader_headers`, by the representation's own +/// `ETag` and `Last-Modified`, see [`MediaClaim::answer_unsigned`]: `412` +/// or `304` with no body, or the response as it is. A range is not applied. +pub(crate) fn answer_whole( + method: &Method, + reader_headers: &HeaderMap, + response: Response, +) -> Response { + answer_whole_with(method, &ReaderPreconditions::of(reader_headers), response) +} + +fn answer_whole_with( + method: &Method, + preconditions: &ReaderPreconditions, + response: Response, +) -> Response { + if response.status() != StatusCode::OK { + return response; + } + let headers = response.headers(); + let etag = headers + .get(header::ETAG) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let last_modified = headers + .get(header::LAST_MODIFIED) + .and_then(|value| value.to_str().ok()) + .and_then(|value| httpdate::parse_http_date(value.trim()).ok()); + let whole = ReaderPreconditions { + range: None, + if_range: None, + ..preconditions.clone() + }; + let outcome = evaluate(method, &whole, etag.as_deref(), last_modified, 0); + let (mut parts, body) = response.into_parts(); + match outcome { + Outcome::NotModified => { + parts.status = StatusCode::NOT_MODIFIED; + for name in ["content-length", "content-range", "transfer-encoding"] { + parts.headers.remove(name); + } + Response::from_parts(parts, EdgeBody::empty()) + } + Outcome::PreconditionFailed => { + parts.status = StatusCode::PRECONDITION_FAILED; + for name in [ + "content-type", + "content-range", + "transfer-encoding", + "content-encoding", + ] { + parts.headers.remove(name); + } + parts + .headers + .insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); + Response::from_parts(parts, EdgeBody::empty()) + } + _ => Response::from_parts(parts, body), + } +} + +/// Removes the headers that describe the origin's bytes or one transfer of +/// them, for a representation core made of them. +pub(crate) fn remove_origin_representation_headers(headers: &mut HeaderMap) { + for name in ORIGIN_REPRESENTATION_HEADERS { + headers.remove(name); + } +} + +impl MediaClaim { + /// Keeps a signed representation for `identity` under `namespace`, the + /// one it was signed under, so a later request is answered before the + /// origin is asked. Whether it was kept. + async fn keep( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + namespace: &str, + value: Bytes, + lifetime: Duration, + ) -> bool { + if !services.loading_cache_for(ASSETS_CACHE).keeps_values() { + return false; + } + let key = self.store_key(services, namespace, identity); + let stored = services + .load_cached( + &self.spec, + &key, + Box::new(move || { + Box::pin(async move { Ok(LoadedValue::new(value).with_lifetime(lifetime)) }) + }), + ) + .await; + match stored { + Ok(_) => true, + Err(error) => { + log::debug!( + "media signing: representation not kept signer={} error={error}", + self.signer.signer_id() + ); + false + } + } + } + + /// The key a signed representation for `identity` is kept under, bound + /// to `namespace`. + fn store_key( + &self, + services: &RuntimeServices, + namespace: &str, + identity: &VariantIdentity<'_>, + ) -> StoreKey { + let mut digest = Sha256::new(); + for part in [ + self.signer.signer_id(), + namespace, + identity.reader_url, + identity.origin_identity, + identity.fetched_url, + ] { + digest.update(part.as_bytes()); + digest.update([0]); + } + StoreKey::new( + services.server_key_prefix(), + ASSETS_CACHE, + format!("media-{}", hex::encode(digest.finalize().as_slice())), + ) + } + + /// Writes `outcome` on the response: its status, its framing headers and + /// the part of `bytes` it carries. + fn apply( + &self, + mut parts: http::response::Parts, + bytes: Bytes, + outcome: Outcome, + ranges_offered: bool, + ) -> Response { + let length = to_u64(bytes.len()); + let headers = &mut parts.headers; + for name in [ + "content-range", + "content-length", + "transfer-encoding", + "content-encoding", + ] { + headers.remove(name); + } + headers.insert( + header::ACCEPT_RANGES, + HeaderValue::from_static(if ranges_offered { "bytes" } else { "none" }), + ); + let carries_body = self.reader_method != Method::HEAD; + let body = match outcome { + Outcome::Full => { + parts.status = StatusCode::OK; + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(length)); + if carries_body { + EdgeBody::Once(bytes) + } else { + EdgeBody::empty() + } + } + Outcome::NotModified => { + parts.status = StatusCode::NOT_MODIFIED; + EdgeBody::empty() + } + Outcome::PreconditionFailed => { + parts.status = StatusCode::PRECONDITION_FAILED; + headers.remove(header::CONTENT_TYPE); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); + EdgeBody::empty() + } + Outcome::Partial { first, last } => { + parts.status = StatusCode::PARTIAL_CONTENT; + headers.insert( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes {first}-{last}/{length}")) + .expect("should build a content range"), + ); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(last - first + 1)); + let from = usize::try_from(first) + .unwrap_or(bytes.len()) + .min(bytes.len()); + let to = usize::try_from(last) + .map_or(bytes.len(), |last| last.saturating_add(1)) + .min(bytes.len()); + EdgeBody::Once(bytes.slice(from..to.max(from))) + } + Outcome::Unsatisfiable => { + parts.status = StatusCode::RANGE_NOT_SATISFIABLE; + headers.insert( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes */{length}")) + .expect("should build a content range"), + ); + headers.remove(header::CONTENT_TYPE); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); + EdgeBody::empty() + } + }; + Response::from_parts(parts, body) + } +} + +/// Origin freshness at receipt. Revalidation-only answers and reader-varying +/// responses authorize no reusable copy. Shared freshness falls back to +/// `max-age` only when `s-maxage` is absent, never when it is malformed. +fn origin_freshness(headers: &HeaderMap, received_at: SystemTime) -> OriginFreshness { + if ["no-store", "no-cache", "private"] + .iter() + .any(|directive| cache_control_has(headers, directive)) + || headers + .get_all(header::VARY) + .iter() + .flat_map(|value| value.to_str().unwrap_or("*").split(',')) + .map(str::trim) + .any(|name| !name.is_empty() && !name.eq_ignore_ascii_case("accept-encoding")) + { + return OriginFreshness::default(); + } + let age = match headers.get(header::AGE) { + Some(value) => match value.to_str().ok().and_then(|value| digits(value.trim())) { + Some(seconds) => Duration::from_secs(seconds), + None => return OriginFreshness::default(), + }, + None => Duration::ZERO, + }; + let apparent_age = match headers.get(header::DATE) { + Some(value) => match value + .to_str() + .ok() + .and_then(|value| httpdate::parse_http_date(value.trim()).ok()) + { + Some(date) => received_at + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .saturating_sub( + date.duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(), + ), + None => return OriginFreshness::default(), + }, + None => Duration::ZERO, + }; + let current_age = age.max(apparent_age); + let browser = cache_control_seconds(headers, "max-age"); + let shared = match cache_control_seconds(headers, "s-maxage") { + Ok(None) => browser, + value => value, + }; + let remaining = |seconds: Result, ()>| { + Duration::from_secs(seconds.ok().flatten().unwrap_or(0)).saturating_sub(current_age) + }; + OriginFreshness { + browser: remaining(browser), + shared: remaining(shared), + } +} + +/// An absent directive permits fallback; an invalid or repeated one does not. +fn cache_control_seconds(headers: &HeaderMap, directive: &str) -> Result, ()> { + let mut seconds = None; + for line in headers.get_all(header::CACHE_CONTROL) { + for part in line.to_str().map_err(|_| ())?.split(',') { + let (name, value) = part.split_once('=').unwrap_or((part, "")); + if !name.trim().eq_ignore_ascii_case(directive) { + continue; + } + if seconds.is_some() { + return Err(()); + } + let value = value.trim(); + let value = if let Some(quoted) = value.strip_prefix('"') { + quoted.strip_suffix('"').ok_or(())? + } else { + value + }; + seconds = Some(digits(value).ok_or(())?); + } + } + Ok(seconds) +} + +/// What a kept signed representation holds besides its key. +struct KeptRecord<'a> { + /// When it was kept, in seconds since the Unix epoch. + kept_at: u64, + /// The `Age` the origin's answer carried when it was kept. + age: u64, + headers: &'a HeaderMap, + body: &'a Bytes, +} + +/// A kept signed representation, as written: [`KEPT_MAGIC`], when it was +/// kept and the age it had then, the header count, each header's name and +/// value with their lengths, then the body. +fn encode_kept(record: &KeptRecord<'_>) -> Bytes { + let mut out = BytesMut::with_capacity(record.body.len() + 512); + out.extend_from_slice(KEPT_MAGIC); + out.extend_from_slice(&record.kept_at.to_be_bytes()); + out.extend_from_slice(&record.age.to_be_bytes()); + let count = u32::try_from(record.headers.len()).unwrap_or(u32::MAX); + out.extend_from_slice(&count.to_be_bytes()); + for (name, value) in record + .headers + .iter() + .take(usize::try_from(count).unwrap_or(usize::MAX)) + { + let name = name.as_str().as_bytes(); + let value = value.as_bytes(); + out.extend_from_slice(&u32::try_from(name.len()).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(name); + out.extend_from_slice(&u32::try_from(value.len()).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(value); + } + out.extend_from_slice(record.body); + out.freeze() +} + +/// A kept signed representation as a `200` answer at `now`, with the `Age` +/// it has reached, or `None` where the bytes are not one [`encode_kept`] +/// wrote or the body is not the one its entity tag names. +fn decode_kept(bytes: &Bytes, now: u64) -> Option> { + let mut reader = KeptReader { + rest: bytes.strip_prefix(KEPT_MAGIC.as_slice())?, + }; + let kept_at = reader.number()?; + let age = reader.number()?; + let count = reader.length()?; + let mut headers = HeaderMap::new(); + for _ in 0..count { + let name_length = reader.length()?; + let name = HeaderName::from_bytes(reader.take(name_length)?).ok()?; + let value_length = reader.length()?; + let value = HeaderValue::from_bytes(reader.take(value_length)?).ok()?; + headers.append(name, value); + } + let body = bytes.slice(bytes.len() - reader.rest.len()..); + let expected = signed_entity_tag(Sha256::digest(&body).as_slice()); + if header_text(&headers, &header::ETAG) != expected { + return None; + } + headers.insert( + header::AGE, + HeaderValue::from(age.saturating_add(now.saturating_sub(kept_at))), + ); + let mut response = Response::new(EdgeBody::Once(body)); + *response.headers_mut() = headers; + Some(response) +} + +/// Reads a kept representation's fields in order. +struct KeptReader<'a> { + rest: &'a [u8], +} + +impl<'a> KeptReader<'a> { + fn take(&mut self, length: usize) -> Option<&'a [u8]> { + let taken = self.rest.get(..length)?; + self.rest = self.rest.get(length..)?; + Some(taken) + } + + fn length(&mut self) -> Option { + let bytes: [u8; 4] = self.take(4)?.try_into().ok()?; + usize::try_from(u32::from_be_bytes(bytes)).ok() + } + + fn number(&mut self) -> Option { + let bytes: [u8; 8] = self.take(8)?.try_into().ok()?; + Some(u64::from_be_bytes(bytes)) + } +} + +/// The fetched address a signer may name as the source, being the address +/// exactly as fetched when it carries no user information, query or +/// fragment, which could hold a credential or a reader's data. +fn source_reference(fetched_url: &str) -> Option<&str> { + let parsed = url::Url::parse(fetched_url).ok()?; + let plain = parsed.username().is_empty() + && parsed.password().is_none() + && parsed.query().is_none() + && parsed.fragment().is_none(); + plain.then_some(fetched_url) +} + +/// Whether `Vary` names `*`, `Cookie` or `Authorization`, so the origin's +/// answer may differ between readers. +fn varies_by_reader(headers: &HeaderMap) -> bool { + headers + .get_all(header::VARY) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .map(str::trim) + .any(|name| { + name == "*" + || name.eq_ignore_ascii_case("cookie") + || name.eq_ignore_ascii_case("authorization") + }) +} + +/// Whether the origin's own `Link` already names a C2PA manifest, so the +/// representation carries provenance core must keep as it came rather +/// than sign again or compete with. +fn carries_manifest_link(headers: &HeaderMap) -> bool { + headers + .get_all(header::LINK) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(';')) + .filter_map(|parameter| { + let (name, relations) = parameter.trim().split_once('=')?; + name.trim().eq_ignore_ascii_case("rel").then_some(relations) + }) + .flat_map(|relations| { + relations.split(|character: char| { + character.is_ascii_whitespace() || character == '"' || character == ',' + }) + }) + .any(|relation| relation.eq_ignore_ascii_case("c2pa-manifest")) +} + +/// The JUMBF description box of a C2PA manifest store: the `jumd` box type +/// followed by the C2PA manifest store UUID, +/// `63327061-0011-0010-8000-00AA00389B71`. Every embedding of a store, a +/// JPEG `APP11` segment, a PNG `caBX` chunk, a WebP `C2PA` chunk, an ISO +/// BMFF `uuid` box or a JPEG XL `jumb` box, carries these bytes +/// contiguously at the head of the store. +const C2PA_STORE_DESCRIPTION: [u8; 20] = [ + b'j', b'u', b'm', b'd', 0x63, 0x32, 0x70, 0x61, 0x00, 0x11, 0x00, 0x10, 0x80, 0x00, 0x00, 0xAA, + 0x00, 0x38, 0x9B, 0x71, +]; + +/// Whether `bytes` carry a C2PA manifest store. This is presence only: +/// whether the store is well placed and intact is the signer's to check, +/// see [`MediaJob::optimizer_output`]. +fn carries_c2pa_manifest_store(bytes: &[u8]) -> bool { + bytes + .windows(C2PA_STORE_DESCRIPTION.len()) + .any(|window| window == C2PA_STORE_DESCRIPTION) +} + +/// The most boxes [`bmff_boxes`] reads at one level before it gives up. +const MAX_BMFF_BOXES: usize = 4096; + +/// The type and body range of each ISO BMFF box laid end to end in +/// `bytes`, or `None` where they are not well formed, or are more than +/// [`MAX_BMFF_BOXES`]. A size of `1` reads a 64-bit size after the type, +/// and a size of `0` runs the box to the end of `bytes`. +fn bmff_boxes(bytes: &[u8]) -> Option)>> { + let mut boxes = Vec::new(); + let mut offset = 0_usize; + while offset < bytes.len() { + if boxes.len() == MAX_BMFF_BOXES { + return None; + } + let header = bytes.get(offset..offset.checked_add(8)?)?; + let size32 = u32::from_be_bytes(header.get(0..4)?.try_into().ok()?); + let kind: [u8; 4] = header.get(4..8)?.try_into().ok()?; + let (header_len, size) = match size32 { + 0 => (8, bytes.len() - offset), + 1 => { + let large = bytes.get(offset + 8..offset.checked_add(16)?)?; + let size = u64::from_be_bytes(large.try_into().ok()?); + (16, usize::try_from(size).ok()?) + } + size => (8, usize::try_from(size).ok()?), + }; + if size < header_len { + return None; + } + let end = offset.checked_add(size)?; + if end > bytes.len() { + return None; + } + boxes.push((kind, offset + header_len..end)); + offset = end; + } + Some(boxes) +} + +/// Whether `bytes` are a part of a fragmented ISO BMFF stream, which is +/// never signed alone: its provenance, such as C2PA verifiable segment +/// info, binds the stream's init segment and media segments together, so +/// a manifest added to one part would compete with the stream's. +/// +/// A media segment starts with `styp`, or has a `moof` before its first +/// `mdat`, and has no `moov`. An init segment starts with `ftyp`, has a +/// `moov` that holds an `mvex`, and has no `mdat`. A progressive file, +/// whose `moov` holds no `mvex` or which carries an `mdat` beside its +/// `moov`, is not a part, and neither is anything not well formed. +fn is_fragmented_stream_part(bytes: &[u8]) -> bool { + let Some(boxes) = bmff_boxes(bytes) else { + return false; + }; + let Some((first, _)) = boxes.first() else { + return false; + }; + let moov = boxes.iter().find(|(kind, _)| kind == b"moov"); + let first_mdat = boxes.iter().position(|(kind, _)| kind == b"mdat"); + let moof_before_mdat = boxes + .iter() + .take(first_mdat.unwrap_or(boxes.len())) + .any(|(kind, _)| kind == b"moof"); + let media_segment = (first == b"styp" || moof_before_mdat) && moov.is_none(); + let init_segment = first == b"ftyp" + && first_mdat.is_none() + && moov.is_some_and(|(_, body)| { + bytes + .get(body.clone()) + .and_then(bmff_boxes) + .is_some_and(|children| children.iter().any(|(kind, _)| kind == b"mvex")) + }); + media_segment || init_segment +} + +/// Whether a `Cache-Control` line names `directive`. +fn cache_control_has(headers: &HeaderMap, directive: &str) -> bool { + headers + .get_all(header::CACHE_CONTROL) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .map(|part| part.split('=').next().unwrap_or_default().trim()) + .any(|name| name.eq_ignore_ascii_case(directive)) +} + +fn header_text<'a>(headers: &'a HeaderMap, name: &HeaderName) -> &'a str { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() +} + +/// The origin's `Content-Length`, where it sent one that parses. +fn declared_length(headers: &HeaderMap) -> Option { + digits(header_text(headers, &header::CONTENT_LENGTH).trim()) +} + +/// The complete length a `206`'s `Content-Range: bytes a-b/total` names. +fn content_range_total(headers: &HeaderMap) -> Option { + let value = header_text(headers, &header::CONTENT_RANGE).trim(); + let (unit, range) = value.split_once(' ')?; + if !unit.eq_ignore_ascii_case("bytes") { + return None; + } + digits(range.rsplit_once('/')?.1.trim()) +} + +fn to_u64(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +/// A decimal number of ASCII digits alone, which `str::parse` would also +/// take with a leading `+`. +fn digits(value: &str) -> Option { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + value.parse().ok() +} + +/// A body read up to a limit. +enum Captured { + /// The whole body, within the limit. + Whole(Bytes), + /// A body over the limit, or one whose stream failed, rebuilt from what + /// was read and what was not, so the reader receives what it would have. + Partial(EdgeBody), +} + +/// Reads `body` whole when it is no larger than `limit`. +async fn capture(body: EdgeBody, limit: usize) -> Captured { + match body { + EdgeBody::Once(bytes) if bytes.len() <= limit => Captured::Whole(bytes), + once @ EdgeBody::Once(_) => Captured::Partial(once), + EdgeBody::Stream(mut stream) => { + let mut chunks = Vec::new(); + let mut total = 0_usize; + while let Some(item) = stream.next().await { + match item { + Ok(chunk) => { + total = total.saturating_add(chunk.len()); + chunks.push(chunk); + if total > limit { + let read = futures::stream::iter(chunks.into_iter().map(Ok)); + return Captured::Partial(EdgeBody::Stream( + read.chain(stream).boxed_local(), + )); + } + } + Err(error) => { + let read = chunks + .into_iter() + .map(Ok) + .chain(std::iter::once(Err(error))); + return Captured::Partial(EdgeBody::Stream( + futures::stream::iter(read).boxed_local(), + )); + } + } + } + let mut whole = BytesMut::with_capacity(total); + for chunk in chunks { + whole.extend_from_slice(&chunk); + } + Captured::Whole(whole.freeze()) + } + } +} + +/// How a request is answered from a signed representation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Outcome { + /// `200` with the whole representation. + Full, + /// `304`. + NotModified, + /// `412`. + PreconditionFailed, + /// `206` with bytes `first` to `last`, inclusive. + Partial { first: u64, last: u64 }, + /// `416`. + Unsatisfiable, +} + +/// Evaluates the reader's preconditions and range against a representation +/// in the order RFC 9110 section 13.2.2 gives. `etag` and `last_modified` +/// are the representation's own validators. A signed representation has no +/// modification date, so it passes `None` and `If-Modified-Since` and +/// `If-Unmodified-Since` are ignored; an `If-Range` date never holds. A +/// `Range` is honored on a `GET` alone, for one range. Several ranges are +/// answered with the whole representation, which the RFC allows. +fn evaluate( + method: &Method, + preconditions: &ReaderPreconditions, + etag: Option<&str>, + last_modified: Option, + length: u64, +) -> Outcome { + let current = etag.and_then(single_entity_tag); + let date_of = + |value: Option<&str>| value.and_then(|value| httpdate::parse_http_date(value.trim()).ok()); + + if let Some(value) = preconditions.if_match.as_deref() { + let matched = match TagList::parse(value) { + TagList::Any => true, + TagList::Tags(tags) => current + .as_ref() + .is_some_and(|current| tags.iter().any(|tag| tag.strong_eq(current))), + }; + if !matched { + return Outcome::PreconditionFailed; + } + } else if let (Some(since), Some(modified)) = ( + date_of(preconditions.if_unmodified_since.as_deref()), + last_modified, + ) && modified > since + { + return Outcome::PreconditionFailed; + } + + if let Some(value) = preconditions.if_none_match.as_deref() { + let matched = match TagList::parse(value) { + TagList::Any => true, + TagList::Tags(tags) => current + .as_ref() + .is_some_and(|current| tags.iter().any(|tag| tag.opaque == current.opaque)), + }; + if matched { + return if *method == Method::GET || *method == Method::HEAD { + Outcome::NotModified + } else { + Outcome::PreconditionFailed + }; + } + } else if (*method == Method::GET || *method == Method::HEAD) + && let (Some(since), Some(modified)) = ( + date_of(preconditions.if_modified_since.as_deref()), + last_modified, + ) + && modified <= since + { + return Outcome::NotModified; + } + + if *method != Method::GET { + return Outcome::Full; + } + let Some(range) = preconditions.range.as_deref() else { + return Outcome::Full; + }; + if let Some(if_range) = preconditions.if_range.as_deref() { + let holds = single_entity_tag(if_range) + .zip(current.as_ref()) + .is_some_and(|(tag, current)| tag.strong_eq(current)); + if !holds { + return Outcome::Full; + } + } + match parse_single_range(range, length) { + RangeRequest::Ignore => Outcome::Full, + RangeRequest::Satisfiable { first, last } => Outcome::Partial { first, last }, + RangeRequest::Unsatisfiable => Outcome::Unsatisfiable, + } +} + +/// One entity tag. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct EntityTag<'a> { + weak: bool, + opaque: &'a str, +} + +impl EntityTag<'_> { + fn strong_eq(&self, other: &EntityTag<'_>) -> bool { + !self.weak && !other.weak && self.opaque == other.opaque + } +} + +/// The entity tag at the start of `input` and what follows it. +fn split_entity_tag(input: &str) -> Option<(EntityTag<'_>, &str)> { + let (weak, quoted) = match input.strip_prefix("W/") { + Some(rest) => (true, rest), + None => (false, input), + }; + let body = quoted.strip_prefix('"')?; + let end = body.find('"')?; + Some(( + EntityTag { + weak, + opaque: &body[..end], + }, + &body[end + 1..], + )) +} + +/// A header holding exactly one entity tag, such as `ETag`. +fn single_entity_tag(value: &str) -> Option> { + let (tag, rest) = split_entity_tag(value.trim())?; + rest.trim().is_empty().then_some(tag) +} + +/// An `If-Match` or `If-None-Match` value. +enum TagList<'a> { + Any, + Tags(Vec>), +} + +impl<'a> TagList<'a> { + /// The tags of a list, as far as it parses. A tag may hold a comma, so + /// the list is read tag by tag. + fn parse(value: &'a str) -> Self { + if value.trim() == "*" { + return Self::Any; + } + let mut tags = Vec::new(); + let mut rest = value; + loop { + rest = rest.trim_start_matches([',', ' ', '\t']); + if rest.is_empty() { + break; + } + let Some((tag, after)) = split_entity_tag(rest) else { + break; + }; + tags.push(tag); + rest = after; + } + Self::Tags(tags) + } +} + +/// What a `Range` header asks of a representation of known length. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RangeRequest { + /// Not one byte range this answers, so the whole representation goes. + Ignore, + Satisfiable { + first: u64, + last: u64, + }, + Unsatisfiable, +} + +fn parse_single_range(value: &str, length: u64) -> RangeRequest { + let Some((unit, set)) = value.trim().split_once('=') else { + return RangeRequest::Ignore; + }; + if !unit.trim().eq_ignore_ascii_case("bytes") { + return RangeRequest::Ignore; + } + let mut specs = set + .split(',') + .map(str::trim) + .filter(|spec| !spec.is_empty()); + let (Some(spec), None) = (specs.next(), specs.next()) else { + return RangeRequest::Ignore; + }; + let Some((first, last)) = spec.split_once('-') else { + return RangeRequest::Ignore; + }; + let (first, last) = (first.trim(), last.trim()); + if first.is_empty() { + let Some(suffix) = digits(last) else { + return RangeRequest::Ignore; + }; + if suffix == 0 || length == 0 { + return RangeRequest::Unsatisfiable; + } + return RangeRequest::Satisfiable { + first: length.saturating_sub(suffix), + last: length - 1, + }; + } + let Some(first) = digits(first) else { + return RangeRequest::Ignore; + }; + let last = if last.is_empty() { + None + } else { + match digits(last) { + Some(last) if last >= first => Some(last), + _ => return RangeRequest::Ignore, + } + }; + if first >= length { + return RangeRequest::Unsatisfiable; + } + RangeRequest::Satisfiable { + first, + last: last.map_or(length - 1, |last| last.min(length - 1)), + } +} + +#[cfg(test)] +pub(crate) mod test_support; + +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/media_signing/large.rs b/crates/trusted-server-core/src/media_signing/large.rs new file mode 100644 index 0000000000..07e2ae355e --- /dev/null +++ b/crates/trusted-server-core/src/media_signing/large.rs @@ -0,0 +1,1582 @@ +//! The large-media lane: a representation larger than a signer's bounded +//! limit, signed without core ever holding it whole. +//! +//! The lane signs an unchanged source only. The bytes the reader receives +//! are the origin's own, so the origin's representation keeps its own +//! validators, length and range handling, and the signer only adds headers, +//! such as a manifest `Link`. Core adds a `Repr-Digest` of the whole source. +//! +//! Core admits an origin answer to the lane only where it can pin one +//! version of the source and keep the outcome: a shared request, an +//! identity-coded answer the origin keeps shareable for a while and lets +//! be transformed, a cache that keeps values, a single strong `ETag`, a +//! known total length above the signer's bounded limit and within its +//! large limit, a platform that streams response bodies and offers a +//! timer, and no C2PA manifest `Link` of the origin's own. Before anything +//! is hashed, core also requires a public address to name the source by: +//! the fetched address where it is plain `https` and core does not sign +//! the request for it, or else the reader's query-free `https` address. +//! An answer that fails any of these goes to the reader as the origin sent +//! it, validators and all; an answer with no strong `ETag` is never given +//! an invented version. Only a reader's `GET` of an answer whose origin +//! says `Accept-Ranges: bytes` starts the hashing of a version; a `HEAD`, +//! or a `GET` without that, is answered from a kept outcome or goes as it +//! came. +//! +//! Every later read of a version being signed, by core or the signer, asks +//! the origin through the route's own backend and origin signing for +//! exactly one byte range with `If-Match` on the pinned tag, and refuses an +//! answer whose status, `Content-Range`, `Content-Length`, coding or `ETag` +//! does not describe exactly that range of exactly that version. A changed +//! version therefore fails the read, never mixes into a digest or a signed +//! delivery. A version the lane declined has no signature to protect, so +//! where the origin's first answer can no longer be sent its reader gets +//! one fresh origin answer to the request as core first sent it, held by +//! no cache. +//! +//! What core keeps for the lane is the outcome only, never the bytes: for +//! a signed version the whole-source digest, the length and the signer's +//! headers, for a declined one (refused, failed or out of time) that it was +//! declined, briefly. Both are kept under the publisher's prefix, the +//! signer's current [`namespace`](super::MediaSigner::namespace), the +//! reader's address, the origin and the pinned tag, for no longer than the +//! origin's shared freshness. A changed or revoked policy names another +//! namespace, so it never finds an outcome kept under the old one. The +//! hashing and signing run as one load of the loading cache, which runs a +//! key's load once for every caller asking for it at the same time. + +use std::fmt; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use base64::Engine as _; +use bytes::{Bytes, BytesMut}; +use edgezero_core::body::Body as EdgeBody; +use futures::StreamExt as _; +use futures::future::Either; +use http::{HeaderMap, HeaderName, HeaderValue, Method, Request, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; +use web_time::Instant; + +use super::{ + Delivery, KeptReader, MediaClaim, MediaSigner, MediaSigningFailure, OriginFacts, Outcome, + VariantIdentity, carries_manifest_link, digits, evaluate, header_text, is_reserved_header, + single_entity_tag, source_reference, strip_representation_preconditions, to_u64, +}; +use crate::middleware::ResponseHeader; +use crate::origin_auth::OriginSigning; +use crate::platform::{ + LoadError, LoadedValue, Loader, PlatformHttpRequest, RuntimeServices, StoreKey, +}; +use crate::settings::ASSETS_CACHE; + +/// What [`MediaSigner::sign_large`](super::MediaSigner::sign_large) returns. +pub type LargeMediaSigning<'a> = + Pin> + 'a>>; + +/// One large, unchanged representation a signer is asked to sign. +#[derive(Debug, Clone, Copy)] +pub struct LargeMediaJob<'a> { + /// The address the reader asked for, `scheme://host/path`, without its + /// query. + pub reader_url: &'a str, + /// The origin's `Content-Type`, as it sent it. + pub content_type: &'a str, + /// The pinned version of the source, read at offsets. + pub source: &'a VersionedMediaSource<'a>, + /// SHA-256 of every byte of the pinned version, lower case hex, which + /// core computed in bounded chunks and checked against its length. The + /// bytes the reader receives are these bytes. + pub source_sha256: &'a str, + /// The address core fetched the source from, exactly as fetched, where + /// it is `https` and carries no user information, query or fragment. + pub source_reference: Option<&'a str>, + /// The reader's address, where it is `https` and the reader's request + /// carried no query. It names the source only because this lane + /// delivers the source unchanged. + pub reader_reference: Option<&'a str>, + /// The lower case host of the address core fetched the source from, + /// with no scheme, port, path, query or user information. + pub source_origin_host: &'a str, + /// What is left of the signer's + /// [`max_large_media_duration`](super::MediaSigner::max_large_media_duration) + /// after core's hashing. Core drops the signing when it is spent. + pub remaining: Duration, +} + +/// What a signer returns for a large representation: headers only, since +/// the bytes are the origin's. +#[derive(Debug, Clone)] +pub struct LargeSignedMedia { + /// Headers written on every answer of the signed version: `200`, `206`, + /// `HEAD` and `304`. A `Link` is added to the origin's own links, and + /// any other replaces the origin's header of that name. A name core + /// owns, see [`is_reserved_header`], fails the signing. + pub headers: Vec, + /// The policy and credential context the source was signed under, as + /// [`MediaSigner::namespace`](super::MediaSigner::namespace) gives it + /// while that context is current. Core keeps the metadata under it. + pub namespace: String, +} + +/// One version of a source: its strong entity tag and its length. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceVersion { + /// The strong tag, quoted, as `"opaque"`. + etag: String, + length: u64, +} + +impl SourceVersion { + /// The version a single strong entity tag `etag` and a length name, or + /// `None` where `etag` is weak, empty or not one tag. + #[must_use] + pub fn new(etag: &str, length: u64) -> Option { + let tag = single_entity_tag(etag)?; + (!tag.weak && !tag.opaque.is_empty()).then(|| Self { + etag: format!("\"{}\"", tag.opaque), + length, + }) + } + + /// The version an origin answer's headers name, where they carry + /// exactly one strong `ETag`. + fn of(headers: &HeaderMap, length: u64) -> Option { + let mut tags = headers.get_all(header::ETAG).iter(); + let (Some(tag), None) = (tags.next(), tags.next()) else { + return None; + }; + Self::new(tag.to_str().ok()?, length) + } + + /// The strong entity tag, quoted. + #[must_use] + pub fn etag(&self) -> &str { + &self.etag + } + + /// The length in bytes. + #[must_use] + pub fn length(&self) -> u64 { + self.length + } + + /// Whether an answer's headers carry this version's tag, strongly. + fn matches(&self, headers: &HeaderMap) -> bool { + let mut tags = headers.get_all(header::ETAG).iter(); + let (Some(tag), None) = (tags.next(), tags.next()) else { + return false; + }; + tag.to_str() + .ok() + .and_then(single_entity_tag) + .is_some_and(|tag| { + !tag.weak && self.etag.get(1..self.etag.len() - 1) == Some(tag.opaque) + }) + } +} + +/// Why a read of a [`VersionedMediaSource`] failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum MediaSourceError { + /// The origin no longer serves the pinned version: it answered `412` to + /// `If-Match`, or with a missing, weak or different `ETag`. Nothing + /// read from this source may be published. + #[display("the origin no longer serves the pinned version")] + VersionChanged, + /// The read reaches past the pinned version's length. + #[display("the read reaches past the pinned version")] + OutOfBounds, + /// The origin answered a pinned read with something other than exactly + /// the bytes asked for. + #[display("the origin answered a pinned read wrongly: {reason}")] + InvalidAnswer { + /// A short code naming what was wrong. + reason: &'static str, + }, + /// The read could not be sent, or its body failed or ended early. + #[display("a pinned read could not be completed")] + Transport, +} + +impl core::error::Error for MediaSourceError {} + +impl MediaSourceError { + /// A short code naming the failure, for logs and refusals. + #[must_use] + pub fn code(&self) -> &'static str { + match self { + Self::VersionChanged => "source_version_changed", + Self::OutOfBounds => "source_read_out_of_bounds", + Self::InvalidAnswer { reason } => reason, + Self::Transport => "source_transport", + } + } +} + +/// How core asks the origin for one more byte range of a source: the +/// address, the request's headers as they stood before it was signed, the +/// backend and the origin's signing. +#[derive(Clone, Copy)] +pub(crate) struct PinnedOrigin<'a> { + /// The address, exactly as fetched and signed. + pub(crate) url: &'a url::Url, + /// The origin request's headers before signing. + pub(crate) headers: &'a HeaderMap, + /// The backend the origin request went to. + pub(crate) backend_name: &'a str, + /// The origin's signing, applied fresh to every read. + pub(crate) auth: Option<&'a OriginSigning>, + /// Headers a signed read must not carry, removed before core writes its + /// own conditions and signs. + pub(crate) unsigned_headers: &'a [HeaderName], + /// Publisher URL mapping for a fresh unsigned response; asset proxies keep + /// the origin's navigation targets as on their ordinary response path. + pub(crate) response_header_rewrite: + Option>, +} + +/// One pinned version of a source, read at offsets from the origin. +/// +/// Every read asks the origin for exactly one byte range with `If-Match` +/// on the pinned tag, through the route's own backend and origin signing, +/// and fails unless the answer is exactly that range of that version. +pub struct VersionedMediaSource<'a> { + services: &'a RuntimeServices, + origin: PinnedOrigin<'a>, + version: SourceVersion, +} + +impl fmt::Debug for VersionedMediaSource<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("VersionedMediaSource") + .field("version", &self.version) + .finish_non_exhaustive() + } +} + +impl<'a> VersionedMediaSource<'a> { + /// A source read from `url` through `backend_name` with `headers`, an + /// origin that signs no request, pinned to `version`. + #[must_use] + pub fn new( + services: &'a RuntimeServices, + url: &'a url::Url, + headers: &'a HeaderMap, + backend_name: &'a str, + version: SourceVersion, + ) -> Self { + Self { + services, + origin: PinnedOrigin { + url, + headers, + backend_name, + auth: None, + unsigned_headers: &[], + response_header_rewrite: None, + }, + version, + } + } + + /// The version every read is pinned to. + #[must_use] + pub fn version(&self) -> &SourceVersion { + &self.version + } + + /// The pinned version's length in bytes. + #[must_use] + pub fn size(&self) -> u64 { + self.version.length + } + + /// Fills `destination` with the bytes at `offset` of the pinned version. + /// + /// # Errors + /// + /// [`MediaSourceError::OutOfBounds`] where the read reaches past the + /// version, [`MediaSourceError::VersionChanged`] where the origin no + /// longer serves it, and the other variants where the origin's answer + /// is not exactly the range asked for or does not arrive whole. + /// + /// # Performance + /// + /// One origin request per call, streamed into `destination`. + pub async fn read_exact_at( + &self, + offset: u64, + destination: &mut [u8], + ) -> Result<(), MediaSourceError> { + let length = to_u64(destination.len()); + let mut filled = 0_usize; + let mut visit = |chunk: &[u8]| { + let end = filled.saturating_add(chunk.len()); + if let Some(slot) = destination.get_mut(filled..end) { + slot.copy_from_slice(chunk); + } + filled = end; + }; + self.stream_range(offset, length, &mut visit).await + } + + /// Hands `visit` the bytes from `offset` for `length` bytes of the + /// pinned version, in order, in the chunks the origin sends. + /// + /// # Errors + /// + /// As [`read_exact_at`](Self::read_exact_at). `visit` may have seen + /// some of the bytes before a read fails. + /// + /// # Performance + /// + /// One origin request; nothing is held beyond one chunk. + pub async fn for_each_chunk( + &self, + offset: u64, + length: u64, + mut visit: F, + ) -> Result<(), MediaSourceError> { + self.stream_range(offset, length, &mut visit).await + } + + /// Whether the origin still serves the pinned version, by a pinned + /// read of its first byte. A signer asks this last before it publishes. + /// + /// # Errors + /// + /// As [`read_exact_at`](Self::read_exact_at). + pub async fn confirm_current(&self) -> Result<(), MediaSourceError> { + let mut first = [0_u8; 1]; + self.read_exact_at(0, &mut first).await + } + + /// The same source read through `origin`, which may sign its reads. + pub(crate) fn pinned( + services: &'a RuntimeServices, + origin: PinnedOrigin<'a>, + version: SourceVersion, + ) -> Self { + Self { + services, + origin, + version, + } + } + + /// Streams `length` bytes from `offset` to `visit`. + async fn stream_range( + &self, + offset: u64, + length: u64, + visit: &mut F, + ) -> Result<(), MediaSourceError> { + if length == 0 { + return Ok(()); + } + let last = offset + .checked_add(length - 1) + .filter(|last| *last < self.version.length) + .ok_or(MediaSourceError::OutOfBounds)?; + let body = self.send_pinned(offset, last).await?; + drain(body, length, visit).await + } + + /// Asks the origin for bytes `first` to `last` of the pinned version + /// and returns the body of an answer that is exactly that. + async fn send_pinned(&self, first: u64, last: u64) -> Result { + let request = self.pinned_request(first, last)?; + // The platform's error may name the address, which can carry a + // credential, so it is not logged. + let answer = self + .services + .http_client() + .send(request) + .await + .map_err(|_| MediaSourceError::Transport)?; + let (parts, body) = answer.response.into_parts(); + check_pinned(&self.version, parts.status, &parts.headers, first, last)?; + Ok(body) + } + + /// The origin request for bytes `first` to `last`: the request as it + /// stood before signing, with every condition the reader sent removed, + /// and only core's own `If-Match`, `Range` and identity coding written + /// before it is signed afresh. + fn pinned_request( + &self, + first: u64, + last: u64, + ) -> Result { + let mut headers = self.origin.base_headers(); + strip_representation_preconditions(&mut headers); + headers.insert( + header::ACCEPT_ENCODING, + HeaderValue::from_static("identity"), + ); + headers.insert( + header::IF_MATCH, + HeaderValue::from_str(&self.version.etag).map_err(|_| { + MediaSourceError::InvalidAnswer { + reason: "etag_not_a_header", + } + })?, + ); + headers.insert( + header::RANGE, + HeaderValue::from_str(&format!("bytes={first}-{last}")) + .expect("should build a byte range"), + ); + let request = self + .origin + .signed_request(&Method::GET, headers) + .map_err(|()| MediaSourceError::Transport)?; + Ok( + if self.services.http_client().supports_streaming_responses() { + request.with_stream_response() + } else { + request.with_max_response_body_bytes( + usize::try_from(last - first + 1).unwrap_or(usize::MAX), + ) + }, + ) + } +} + +impl PinnedOrigin<'_> { + /// The origin request's headers as they stood before signing, less + /// the headers a signed request must not carry where core signs it. + fn base_headers(&self) -> HeaderMap { + let mut headers = self.headers.clone(); + if self.auth.is_some() { + for name in self.unsigned_headers { + headers.remove(name); + } + } + headers + } + + /// A `method` request to the origin carrying `headers`, signed afresh + /// where the origin signs requests. Nothing is written after the + /// signature. + fn signed_request( + &self, + method: &Method, + mut headers: HeaderMap, + ) -> Result { + if let Some(auth) = self.auth { + auth.remove_namespace_headers(&mut headers); + auth.sign(method, self.url, &mut headers).map_err(|_| ())?; + } + let mut request = Request::builder() + .method(method.clone()) + .uri(self.url.as_str()) + .body(EdgeBody::empty()) + .map_err(|_| ())?; + *request.headers_mut() = headers; + Ok(PlatformHttpRequest::new(request, self.backend_name)) + } +} + +/// Checks that an answer to a pinned read of bytes `first` to `last` is +/// exactly that range of `version`. +fn check_pinned( + version: &SourceVersion, + status: StatusCode, + headers: &HeaderMap, + first: u64, + last: u64, +) -> Result<(), MediaSourceError> { + if status == StatusCode::PRECONDITION_FAILED { + return Err(MediaSourceError::VersionChanged); + } + if status != StatusCode::OK && status != StatusCode::PARTIAL_CONTENT { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_status", + }); + } + if !version.matches(headers) { + return Err(MediaSourceError::VersionChanged); + } + let encoding = header_text(headers, &header::CONTENT_ENCODING).trim(); + if !encoding.is_empty() && !encoding.eq_ignore_ascii_case("identity") { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_content_encoded", + }); + } + let whole = first == 0 && last.checked_add(1) == Some(version.length); + if status == StatusCode::PARTIAL_CONTENT { + if content_range(headers) != Some((first, last, version.length)) { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_content_range", + }); + } + } else if !whole { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_status", + }); + } + let declared = header_text(headers, &header::CONTENT_LENGTH).trim(); + if !declared.is_empty() && digits(declared) != Some(last - first + 1) { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_content_length", + }); + } + Ok(()) +} + +/// The first byte, last byte and complete length a `206`'s +/// `Content-Range: bytes first-last/length` names. +fn content_range(headers: &HeaderMap) -> Option<(u64, u64, u64)> { + let value = header_text(headers, &header::CONTENT_RANGE).trim(); + let (unit, range) = value.split_once(' ')?; + if !unit.eq_ignore_ascii_case("bytes") { + return None; + } + let (span, total) = range.trim().split_once('/')?; + let (first, last) = span.split_once('-')?; + Some(( + digits(first.trim())?, + digits(last.trim())?, + digits(total.trim())?, + )) +} + +/// Hands `visit` every chunk of `body`, which must hold exactly `expected` +/// bytes. A chunk that would pass `expected` is refused before `visit` +/// sees it. A stream that fails or ends early is a +/// [`MediaSourceError::Transport`]. +async fn drain( + body: EdgeBody, + expected: u64, + visit: &mut F, +) -> Result<(), MediaSourceError> { + match body { + EdgeBody::Once(bytes) => { + if to_u64(bytes.len()) != expected { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_body_length", + }); + } + visit(&bytes); + Ok(()) + } + EdgeBody::Stream(mut stream) => { + let mut seen = 0_u64; + while let Some(item) = stream.next().await { + let Ok(chunk) = item else { + return Err(MediaSourceError::Transport); + }; + let next = seen.saturating_add(to_u64(chunk.len())); + if next > expected { + return Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_overlong_body", + }); + } + visit(&chunk); + seen = next; + } + if seen == expected { + Ok(()) + } else { + Err(MediaSourceError::Transport) + } + } + } +} + +/// SHA-256 of every byte of `source`'s pinned version, read once by one +/// pinned read in bounded chunks. A read that fails fails the digest; it is +/// never restarted or resumed. +async fn whole_digest(source: &VersionedMediaSource<'_>) -> Result<[u8; 32], MediaSourceError> { + let mut hasher = Sha256::new(); + let mut visit = |chunk: &[u8]| hasher.update(chunk); + source.stream_range(0, source.size(), &mut visit).await?; + let mut digest = [0_u8; 32]; + digest.copy_from_slice(&hasher.finalize()); + Ok(digest) +} + +/// `body` as sent to the reader, refused where it holds more or fewer than +/// `expected` bytes, so a reader never receives bytes of another length +/// under headers that describe this one. +fn exact_body(body: EdgeBody, expected: u64) -> Result { + match body { + EdgeBody::Once(bytes) if to_u64(bytes.len()) == expected => Ok(EdgeBody::Once(bytes)), + EdgeBody::Once(_) => Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_body_length", + }), + EdgeBody::Stream(stream) => Ok(EdgeBody::from_stream(futures::stream::unfold( + (stream, 0_u64, false), + move |(mut stream, sent, finished)| async move { + if finished { + return None; + } + match stream.next().await { + Some(Ok(chunk)) => { + let sent = sent.saturating_add(to_u64(chunk.len())); + if sent > expected { + let error = MediaSourceError::InvalidAnswer { + reason: "pinned_read_overlong_body", + }; + Some((Err(error), (stream, sent, true))) + } else { + Some((Ok(chunk), (stream, sent, false))) + } + } + Some(Err(_)) => Some((Err(MediaSourceError::Transport), (stream, sent, true))), + None if sent == expected => None, + None => Some((Err(MediaSourceError::Transport), (stream, sent, true))), + } + }, + ))), + } +} + +/// The first bytes of kept metadata for a signed version, naming its +/// layout. +const SIGNED_MAGIC: &[u8; 4] = b"TSL1"; + +/// The whole of the kept record for a version the lane declined: refused +/// by the signer, failed to read or hash, or out of time. +const DECLINED_MAGIC: &[u8; 4] = b"TSL0"; + +/// The longest metadata for a signed version is kept, whatever the origin +/// allows. +const SIGNED_MAX_LIFETIME: Duration = Duration::from_secs(3600); + +/// The longest a declined version is kept, so a source, policy or signer +/// that changes is tried again soon, while a burst of requests for one +/// version never hashes it once per request. +const DECLINED_MAX_LIFETIME: Duration = Duration::from_secs(60); + +/// How long a lookup may take before the origin's own answer, held unread +/// meanwhile, is no longer trusted to stream, and the bytes are read again +/// pinned. A lookup that waits on another caller's load can take as long as +/// that load. +const ORIGIN_BODY_MAX_WAIT: Duration = Duration::from_secs(1); + +/// The most signer headers kept metadata may hold. +const LARGE_MAX_HEADERS: usize = 64; + +/// What core keeps for a signed large version: never its bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +struct LargeRecord { + length: u64, + sha256: [u8; 32], + headers: Vec, +} + +/// What core keeps for one large version under one namespace. +#[derive(Debug, Clone, PartialEq, Eq)] +enum KeptOutcome { + /// The version was signed; its answers carry these headers. + Signed(LargeRecord), + /// The version was not signed; its answers go as the origin made them. + Declined, +} + +/// Metadata for a signed version as written: [`SIGNED_MAGIC`], the length, +/// the digest, the header count, then each header's name and value with +/// their lengths. +fn encode_large(record: &LargeRecord) -> Bytes { + let mut out = BytesMut::with_capacity(128); + out.extend_from_slice(SIGNED_MAGIC); + out.extend_from_slice(&record.length.to_be_bytes()); + out.extend_from_slice(&record.sha256); + let count = record.headers.len().min(LARGE_MAX_HEADERS); + out.extend_from_slice(&u32::try_from(count).unwrap_or(u32::MAX).to_be_bytes()); + for (name, value) in record.headers.iter().take(count) { + for field in [name.as_str().as_bytes(), value.as_bytes()] { + out.extend_from_slice(&u32::try_from(field.len()).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(field); + } + } + out.freeze() +} + +/// A kept outcome as [`encode_large`] or [`DECLINED_MAGIC`] wrote it, or +/// `None` for any other bytes. +fn decode_kept_outcome(bytes: &[u8]) -> Option { + if bytes == DECLINED_MAGIC.as_slice() { + return Some(KeptOutcome::Declined); + } + let mut reader = KeptReader { + rest: bytes.strip_prefix(SIGNED_MAGIC.as_slice())?, + }; + let length = reader.number()?; + let sha256: [u8; 32] = reader.take(32)?.try_into().ok()?; + let count = reader.length()?; + if count > LARGE_MAX_HEADERS { + return None; + } + let mut headers = Vec::with_capacity(count); + for _ in 0..count { + let name_length = reader.length()?; + let name = HeaderName::from_bytes(reader.take(name_length)?).ok()?; + let value_length = reader.length()?; + let value = HeaderValue::from_bytes(reader.take(value_length)?).ok()?; + if is_reserved_header(&name) { + return None; + } + headers.push((name, value)); + } + reader + .rest + .is_empty() + .then_some(KeptOutcome::Signed(LargeRecord { + length, + sha256, + headers, + })) +} + +/// Writes a signed version's headers on an answer of it: the signer's, +/// with a `Link` added to the origin's, and the whole source's +/// `Repr-Digest`. +fn decorate(headers: &mut HeaderMap, record: &LargeRecord) { + for (name, value) in &record.headers { + if name == header::LINK { + headers.append(name.clone(), value.clone()); + } else { + headers.insert(name.clone(), value.clone()); + } + } + let digest = format!( + "sha-256=:{}:", + base64::engine::general_purpose::STANDARD.encode(record.sha256) + ); + headers.insert( + HeaderName::from_static("repr-digest"), + HeaderValue::from_str(&digest).expect("should build a base64 digest"), + ); +} + +/// The answer where a signed or meant-to-be-signed version can no longer +/// be read: nothing of another version is sent in its place. +fn refusal() -> Response { + Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .header(header::CACHE_CONTROL, "no-store") + .header(header::RETRY_AFTER, "1") + .header(header::CONTENT_LENGTH, 0) + .body(EdgeBody::empty()) + .expect("should build a refusal") +} + +/// The headers a fresh origin answer for an unsigned version brings: its +/// status framing, type, coding, validators and redirect/authentication +/// semantics. Every other header stays as core prepared it for the reader. +const FRESH_REPRESENTATION_HEADERS: [&str; 16] = [ + "content-type", + "content-location", + "content-length", + "content-encoding", + "content-range", + "accept-ranges", + "etag", + "last-modified", + "repr-digest", + "content-digest", + "digest", + "content-md5", + "transfer-encoding", + "location", + "www-authenticate", + "retry-after", +]; + +/// Whether the origin says it serves byte ranges of this representation, +/// which every pinned read relies on. +fn advertises_byte_ranges(headers: &HeaderMap) -> bool { + headers + .get_all(header::ACCEPT_RANGES) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .any(|unit| unit.trim().eq_ignore_ascii_case("bytes")) +} + +/// The answer where the origin could not be asked again for an unsigned +/// version. +fn origin_unreachable() -> Response { + Response::builder() + .status(StatusCode::BAD_GATEWAY) + .header(header::CACHE_CONTROL, "no-store") + .header(header::CONTENT_LENGTH, 0) + .body(EdgeBody::empty()) + .expect("should build a gateway failure") +} + +/// An origin answer admitted to the large lane, with the version it pins. +pub(crate) struct LargeAnswer { + response: Response, + version: SourceVersion, + /// How long the origin let its answer be kept, shared, which bounds + /// how long its outcome is kept. + keep_for: Duration, +} + +impl LargeAnswer { + /// The origin's answer as it came, for a caller that cannot say how to + /// ask the origin again, sent unsigned. + pub(crate) fn into_origin_response(self) -> Response { + self.response + } +} + +/// The one wall-clock budget for a large source's namespace resolution, +/// cache waits, hashing and signing. +#[derive(Debug, Clone, Copy)] +struct LargeBudget { + started: Instant, + total: Duration, +} + +impl LargeBudget { + /// What is left of the budget now. + fn remaining(self) -> Duration { + self.total.saturating_sub(self.started.elapsed()) + } +} + +/// The public addresses a large source may be named by, worked out before +/// anything is hashed. +#[derive(Debug, Clone, PartialEq, Eq)] +struct LargeReferences { + /// The fetched address, where it is `https`, carries no user + /// information, query or fragment, and core does not sign the request + /// for it: an address core signs is a private one and is never named. + source: Option, + /// The reader's address, where it is `https` and the reader's request + /// carried no query. + reader: Option, +} + +impl LargeReferences { + fn of(identity: &VariantIdentity<'_>, origin_signed: bool) -> Self { + let source = if origin_signed { + None + } else { + source_reference(identity.fetched_url) + .filter(|reference| reference.starts_with("https://")) + .map(str::to_owned) + }; + let reader = (identity.reader_query_free && identity.reader_url.starts_with("https://")) + .then(|| identity.reader_url.to_owned()); + Self { source, reader } + } + + fn any(&self) -> bool { + self.source.is_some() || self.reader.is_some() + } +} + +/// Everything the hashing and signing of one large version owns, because +/// the loading cache keeps the load with its key rather than borrowing the +/// request, and runs it once for every caller asking for that key at the +/// same time. +struct LargeLoad { + signer: Arc, + /// The request's services without its connection, since the outcome + /// serves every caller waiting on the load. + services: RuntimeServices, + url: url::Url, + headers: HeaderMap, + backend_name: String, + auth: Option, + unsigned_headers: Vec, + version: SourceVersion, + reader_url: String, + content_type: String, + references: LargeReferences, + /// The namespace the outcome is kept under, which the signer must sign + /// under. + namespace: String, + budget: LargeBudget, + keep_for: Duration, +} + +impl LargeLoad { + fn into_loader(self) -> Loader { + Box::new(move || Box::pin(async move { Ok(self.run().await) })) + } + + /// Hashes and signs within the budget, giving the outcome to keep. A + /// declined outcome is kept briefly too, so the version is not hashed + /// again for every request while it stands. + async fn run(self) -> LoadedValue { + let signer_id = self.signer.signer_id(); + let remaining = self.budget.remaining(); + let outcome = match self.services.timer() { + Some(_) if remaining.is_zero() => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + None + } + Some(timer) => { + match futures::future::select(Box::pin(self.sign()), timer.sleep(remaining)).await { + Either::Left((outcome, _)) => outcome, + Either::Right(((), _)) => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + None + } + } + } + None => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=no_timer" + ); + None + } + }; + match outcome { + Some(record) => LoadedValue::new(encode_large(&record)) + .with_lifetime(self.keep_for.min(SIGNED_MAX_LIFETIME)), + None => LoadedValue::new(Bytes::from_static(DECLINED_MAGIC)) + .with_lifetime(self.keep_for.min(DECLINED_MAX_LIFETIME)), + } + } + + /// Hashes the pinned version once and asks the signer for it. + async fn sign(&self) -> Option { + let signer_id = self.signer.signer_id(); + let declined = |reason: &str| { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason={reason}" + ); + }; + let origin = PinnedOrigin { + url: &self.url, + headers: &self.headers, + backend_name: &self.backend_name, + auth: self.auth.as_ref(), + unsigned_headers: &self.unsigned_headers, + response_header_rewrite: None, + }; + let source = VersionedMediaSource::pinned(&self.services, origin, self.version.clone()); + let digest = whole_digest(&source) + .await + .map_err(|error| declined(error.code())) + .ok()?; + let Some(origin_host) = self.url.host_str().map(str::to_ascii_lowercase) else { + declined("no_origin_host"); + return None; + }; + let remaining = self.budget.remaining(); + if remaining.is_zero() { + declined("timeout"); + return None; + } + let source_sha256 = hex::encode(digest); + let job = LargeMediaJob { + reader_url: &self.reader_url, + content_type: &self.content_type, + source: &source, + source_sha256: &source_sha256, + source_reference: self.references.source.as_deref(), + reader_reference: self.references.reader.as_deref(), + source_origin_host: &origin_host, + remaining, + }; + let signed = match self.signer.sign_large(&self.services, &job).await { + Ok(signed) + if signed + .headers + .iter() + .any(|(name, _)| is_reserved_header(name)) => + { + declined("reserved_header"); + return None; + } + Ok(signed) if signed.headers.len() > LARGE_MAX_HEADERS => { + declined("too_many_headers"); + return None; + } + Ok(signed) if signed.namespace != self.namespace => { + declined("namespace_changed"); + return None; + } + Ok(signed) => signed, + Err(failure) => { + declined(failure.reason); + return None; + } + }; + log::info!( + "media signing: outcome=signed lane=large signer={signer_id} source_bytes={}", + source.size() + ); + Some(LargeRecord { + length: source.size(), + sha256: digest, + headers: signed.headers, + }) + } +} + +impl MediaClaim { + /// Whether this claim may use the large lane at all: the signer signs + /// large sources and the request is shared. + fn large_lane_open(&self) -> bool { + self.shared && self.signer.max_large_source_bytes() > 0 + } + + /// The version a signable origin answer of `total` bytes, larger than + /// the bounded limit, pins in the large lane, or why it cannot. The + /// answer's coding, privacy, transform rule, type and any manifest + /// `Link` of the origin's own were already checked, see + /// `unsignable_reason`. Every check here is cheap and runs before + /// anything is hashed. + fn large_version( + &self, + services: &RuntimeServices, + headers: &HeaderMap, + total: Option, + facts: &OriginFacts, + ) -> Result { + let ceiling = self.signer.max_large_source_bytes(); + if ceiling == 0 { + return Err("exceeds_limit"); + } + if !self.shared { + return Err("large_request_not_shared"); + } + // An outcome that cannot be kept would be worked out again for + // every request, each range of one playback included. + if facts.keep_for.is_none() || !services.loading_cache_for(ASSETS_CACHE).keeps_values() { + return Err("large_outcome_not_kept"); + } + let Some(length) = total else { + return Err("unknown_length"); + }; + if length > ceiling { + return Err("exceeds_large_limit"); + } + if !services.http_client().supports_streaming_responses() { + return Err("no_streaming_transport"); + } + if self.signer.max_large_media_duration().is_zero() { + return Err("no_large_media_budget"); + } + if services.timer().is_none() { + return Err("no_timer"); + } + SourceVersion::of(headers, length).ok_or("no_strong_etag") + } + + /// What a signable origin answer of `total` bytes, larger than the + /// bounded limit, becomes: a [`Delivery::Large`] where the large lane + /// admits it, else the origin's answer as it came. + pub(super) fn large_or_unsigned( + &self, + services: &RuntimeServices, + response: Response, + total: Option, + facts: &OriginFacts, + ) -> Delivery { + match self.large_version(services, response.headers(), total, facts) { + Ok(version) => Delivery::Large(LargeAnswer { + response, + version, + keep_for: facts.keep_for.unwrap_or_default(), + }), + Err(reason) => { + log::info!( + "media signing: outcome=unsigned signer={} reason={reason}", + self.signer.signer_id() + ); + Delivery::Send(response) + } + } + } + + /// Answers the reader from an origin answer admitted to the large lane. + /// + /// A source no public address can name, or whose signer's namespace + /// cannot be resolved, goes as the origin answered it, before anything + /// is hashed. An outcome kept for the pinned version under the signer's + /// current namespace answers next: a signed one decorates the origin's + /// own answer, a declined one leaves it as it came. A `HEAD`, or an + /// answer whose origin does not say it serves byte ranges, never starts + /// a load and goes as it came. Otherwise the version is hashed and + /// signed once, through the loading cache, so callers asking at the + /// same time share that one load. A signed version is answered by + /// pinned reads, refused where the origin no longer serves it; a + /// declined one by one fresh origin answer to the request as core first + /// sent it, since no signature binds its bytes. `origin` is how the + /// origin is asked again. + pub(crate) async fn deliver_large( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + origin: PinnedOrigin<'_>, + answer: LargeAnswer, + ) -> Response { + let signer_id = self.signer.signer_id(); + let budget = LargeBudget { + started: Instant::now(), + total: self.signer.max_large_media_duration(), + }; + let LargeAnswer { + response, + version, + keep_for, + } = answer; + let references = LargeReferences::of(identity, origin.auth.is_some()); + if !references.any() { + log::info!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=no_public_reference" + ); + return response; + } + let Some(timer) = services.timer() else { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=no_timer" + ); + return response; + }; + let remaining = budget.remaining(); + if remaining.is_zero() { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + return response; + } + let namespace = match futures::future::select( + self.signer.namespace(services), + timer.sleep(remaining), + ) + .await + { + Either::Left((Ok(namespace), _)) => namespace, + Either::Left((Err(failure), _)) => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason={failure}" + ); + return response; + } + Either::Right(((), _)) => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + return response; + } + }; + let key = self.large_store_key(services, &namespace, identity, version.etag()); + let looked_up = Instant::now(); + let kept = match futures::future::select( + Box::pin(self.large_kept(services, &key, Some(version.length()))), + timer.sleep(budget.remaining()), + ) + .await + { + Either::Left((kept, _)) => kept, + Either::Right(((), _)) => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + if looked_up.elapsed() <= ORIGIN_BODY_MAX_WAIT { + return response; + } + let (parts, _) = response.into_parts(); + return self.deliver_fresh(services, origin, parts).await; + } + }; + if let Some(kept) = kept { + if looked_up.elapsed() <= ORIGIN_BODY_MAX_WAIT { + log::debug!("media signing: lane=large signer={signer_id} from=kept_outcome"); + return match kept { + KeptOutcome::Signed(record) => { + let (mut parts, body) = response.into_parts(); + decorate(&mut parts.headers, &record); + Response::from_parts(parts, body) + } + KeptOutcome::Declined => response, + }; + } + // The origin's answer waited unread for the lookup, so it is no + // longer trusted to stream. + let (parts, _) = response.into_parts(); + return match kept { + KeptOutcome::Signed(record) => { + let source = VersionedMediaSource::pinned(services, origin, version); + self.deliver_pinned(&source, parts, &record).await + } + KeptOutcome::Declined => self.deliver_fresh(services, origin, parts).await, + }; + } + // Only a reader's own `GET` of a version the origin serves in byte + // ranges starts the work of hashing it. + if self.reader_method == Method::HEAD { + log::debug!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=head_starts_no_load" + ); + return response; + } + if !advertises_byte_ranges(response.headers()) { + log::info!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=origin_ranges_not_advertised" + ); + return response; + } + if budget.remaining().is_zero() { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + if looked_up.elapsed() <= ORIGIN_BODY_MAX_WAIT { + return response; + } + let (parts, _) = response.into_parts(); + return self.deliver_fresh(services, origin, parts).await; + } + // The origin's own body is not read: the load reads the version + // pinned, so it serves every caller waiting on it. + let (parts, _) = response.into_parts(); + let load = LargeLoad { + signer: Arc::clone(&self.signer), + services: services.clone().without_connection(), + url: origin.url.clone(), + headers: origin.headers.clone(), + backend_name: origin.backend_name.to_owned(), + auth: origin.auth.cloned(), + unsigned_headers: origin.unsigned_headers.to_vec(), + version: version.clone(), + reader_url: identity.reader_url.to_owned(), + content_type: header_text(&parts.headers, &header::CONTENT_TYPE).to_owned(), + references, + namespace, + budget, + keep_for, + }; + let loaded = match futures::future::select( + Box::pin(services.load_cached(&self.spec, &key, load.into_loader())), + timer.sleep(budget.remaining()), + ) + .await + { + Either::Left((loaded, _)) => loaded.ok().and_then(|bytes| decode_kept_outcome(&bytes)), + Either::Right(((), _)) => { + log::warn!( + "media signing: outcome=unsigned lane=large signer={signer_id} reason=timeout" + ); + None + } + }; + match loaded { + Some(KeptOutcome::Signed(record)) if record.length == version.length() => { + let source = VersionedMediaSource::pinned(services, origin, version); + self.deliver_pinned(&source, parts, &record).await + } + _ => self.deliver_fresh(services, origin, parts).await, + } + } + + /// Answers the reader of a version the lane does not sign with one + /// fresh origin answer to the request as core first sent it, the + /// reader's own range and conditions included, its origin signing + /// renewed. Nothing is retried. + /// + /// The status, body and every header describing the representation + /// are the fresh answer's, so a changed version goes with its own + /// validators. The rest are `parts`, the headers core and its caller + /// prepared for this reader, to which the fresh answer's `Link` and + /// `Content-Security-Policy` values are added, never replacing the + /// caller's: provenance the origin now declares is kept, and several + /// policies only narrow what a reader allows. The caller's cache policy + /// describes the first answer, not this one, so the answer is held by + /// no cache at all. A cookie or any other header the fresh answer + /// sets is never passed on, since the caller's own handling never saw + /// it. + async fn deliver_fresh( + &self, + services: &RuntimeServices, + origin: PinnedOrigin<'_>, + mut parts: http::response::Parts, + ) -> Response { + let signer_id = self.signer.signer_id(); + let Ok(mut request) = origin.signed_request(&self.reader_method, origin.base_headers()) + else { + log::warn!( + "media signing: outcome=unanswered lane=large signer={signer_id} reason=origin_request_unsigned" + ); + return origin_unreachable(); + }; + if services.http_client().supports_streaming_responses() { + request = request.with_stream_response(); + } + // The platform's error may name the address, which can carry a + // credential, so it is not logged. + let Ok(fresh) = services.http_client().send(request).await else { + log::warn!( + "media signing: outcome=unanswered lane=large signer={signer_id} reason=origin_unreachable" + ); + return origin_unreachable(); + }; + let (mut fresh, body) = fresh.response.into_parts(); + // Fresh cookies are not delivered. The existing redirect-loop guard + // must not assume they will change the reader's next request. + fresh.headers.remove(header::SET_COOKIE); + if let Some(rewrite) = origin.response_header_rewrite { + crate::response_header_rewrite::rewrite_origin_urls_in_headers( + &mut fresh.headers, + &rewrite, + ); + } + parts.status = fresh.status; + for name in FRESH_REPRESENTATION_HEADERS { + parts.headers.remove(name); + for value in fresh.headers.get_all(name) { + parts.headers.append(name, value.clone()); + } + } + for name in [header::LINK, header::CONTENT_SECURITY_POLICY] { + for value in fresh.headers.get_all(&name) { + if !parts + .headers + .get_all(&name) + .iter() + .any(|kept| kept == value) + { + parts.headers.append(name.clone(), value.clone()); + } + } + } + let mut response = Response::from_parts(parts, body); + crate::cache_policy::remove_owned_cache_headers(&mut response); + let headers = response.headers_mut(); + crate::cache_policy::apply_no_store_private_to_headers(headers); + for name in [header::EXPIRES, header::AGE] { + headers.remove(name); + } + response + } + + /// Decorates the origin's `304` for a large version kept as signed + /// under the signer's current namespace, so a reader revalidating it is + /// told of its manifest. Any other `304` goes as it came. + pub(super) async fn decorate_not_modified( + &self, + services: &RuntimeServices, + identity: &VariantIdentity<'_>, + facts: &OriginFacts, + response: Response, + ) -> Response { + // A `304` whose own `Link` already names a manifest keeps the + // origin's provenance as it came; no kept outcome is added to it. + if !self.large_lane_open() + || !self.looks_like_media + || facts.private + || carries_manifest_link(response.headers()) + { + return response; + } + let Some(version) = SourceVersion::of(response.headers(), 0) else { + return response; + }; + let Some(timer) = services.timer() else { + return response; + }; + let budget = LargeBudget { + started: Instant::now(), + total: self.signer.max_large_media_duration(), + }; + let remaining = budget.remaining(); + if remaining.is_zero() { + return response; + } + let namespace = + match futures::future::select(self.signer.namespace(services), timer.sleep(remaining)) + .await + { + Either::Left((Ok(namespace), _)) => namespace, + _ => return response, + }; + let key = self.large_store_key(services, &namespace, identity, version.etag()); + let remaining = budget.remaining(); + if remaining.is_zero() { + return response; + } + let record = match futures::future::select( + Box::pin(self.large_kept(services, &key, None)), + timer.sleep(remaining), + ) + .await + { + Either::Left((Some(KeptOutcome::Signed(record)), _)) => record, + _ => return response, + }; + let (mut parts, body) = response.into_parts(); + decorate(&mut parts.headers, &record); + Response::from_parts(parts, body) + } + + /// Answers the reader's preconditions and range against the pinned, + /// signed version, reading the bytes it needs by pinned reads, on + /// `parts`, the origin's headers as core prepared them, and adds the + /// signed version's headers from `record`. A version the origin no + /// longer serves is refused, never replaced, since the headers bind + /// these exact bytes. + async fn deliver_pinned( + &self, + source: &VersionedMediaSource<'_>, + mut parts: http::response::Parts, + record: &LargeRecord, + ) -> Response { + let length = source.size(); + let outcome = evaluate( + &self.reader_method, + &self.preconditions, + Some(source.version().etag()), + None, + length, + ); + let carries_body = self.reader_method != Method::HEAD; + let range = match outcome { + Outcome::Full if carries_body => length.checked_sub(1).map(|last| (0, last)), + Outcome::Partial { first, last } => Some((first, last)), + _ => None, + }; + let body = match range { + Some((first, last)) => { + let read = source.send_pinned(first, last).await; + match read.and_then(|body| exact_body(body, last - first + 1)) { + Ok(body) => body, + Err(error) => { + log::warn!( + "media signing: outcome=refused lane=large signer={} reason={}", + self.signer.signer_id(), + error.code() + ); + return refusal(); + } + } + } + None => EdgeBody::empty(), + }; + let headers = &mut parts.headers; + for name in [ + "content-range", + "content-length", + "transfer-encoding", + "content-encoding", + ] { + headers.remove(name); + } + match outcome { + Outcome::Full => { + parts.status = StatusCode::OK; + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(length)); + } + Outcome::Partial { first, last } => { + parts.status = StatusCode::PARTIAL_CONTENT; + headers.insert( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes {first}-{last}/{length}")) + .expect("should build a content range"), + ); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(last - first + 1)); + } + Outcome::NotModified => { + parts.status = StatusCode::NOT_MODIFIED; + } + Outcome::PreconditionFailed => { + parts.status = StatusCode::PRECONDITION_FAILED; + headers.remove(header::CONTENT_TYPE); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); + } + Outcome::Unsatisfiable => { + parts.status = StatusCode::RANGE_NOT_SATISFIABLE; + headers.insert( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes */{length}")) + .expect("should build a content range"), + ); + headers.remove(header::CONTENT_TYPE); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); + } + } + if matches!( + outcome, + Outcome::Full | Outcome::Partial { .. } | Outcome::NotModified + ) { + decorate(headers, record); + } + Response::from_parts(parts, body) + } + + /// The outcome kept under `key`, or `None`. A signed outcome whose + /// length is not `length`, where an answer states one, or a record that + /// does not read back, is dropped. The lookup itself runs no load, but + /// waits on one another caller is running for `key`. + async fn large_kept( + &self, + services: &RuntimeServices, + key: &StoreKey, + length: Option, + ) -> Option { + if !services.loading_cache_for(ASSETS_CACHE).keeps_values() { + return None; + } + let bytes = services + .load_cached( + &self.spec, + key, + Box::new(|| Box::pin(async { Err(LoadError::new(super::NOT_KEPT)) })), + ) + .await + .ok()?; + let kept = decode_kept_outcome(&bytes).filter(|kept| match kept { + KeptOutcome::Signed(record) => length.is_none_or(|length| record.length == length), + KeptOutcome::Declined => true, + }); + if kept.is_none() { + log::warn!( + "media signing: lane=large kept outcome unreadable or for another length signer={}", + self.signer.signer_id() + ); + services + .loading_cache_for(ASSETS_CACHE) + .remove(&services.clone().without_connection(), &self.spec, key) + .await; + } + kept + } + + /// The key the outcome for `identity`'s version `etag` is kept under, + /// bound to the publisher's prefix, the signer and its `namespace`. + fn large_store_key( + &self, + services: &RuntimeServices, + namespace: &str, + identity: &VariantIdentity<'_>, + etag: &str, + ) -> StoreKey { + let mut digest = Sha256::new(); + for part in [ + self.signer.signer_id(), + namespace, + identity.reader_url, + identity.origin_identity, + identity.fetched_url, + etag, + ] { + digest.update(part.as_bytes()); + digest.update([0]); + } + StoreKey::new( + services.server_key_prefix(), + ASSETS_CACHE, + format!("media-large-{}", hex::encode(digest.finalize().as_slice())), + ) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/media_signing/large/tests.rs b/crates/trusted-server-core/src/media_signing/large/tests.rs new file mode 100644 index 0000000000..d6e72274f7 --- /dev/null +++ b/crates/trusted-server-core/src/media_signing/large/tests.rs @@ -0,0 +1,2020 @@ +use std::sync::{Arc, Mutex}; +use std::time::SystemTime; + +use base64::Engine as _; +use error_stack::Report; +use futures::FutureExt as _; +use futures::StreamExt as _; +use futures::channel::oneshot; +use futures::executor::block_on; +use sha2::Digest as _; + +use super::*; +use crate::error::TrustedServerError; +use crate::media_signing::test_support::MANIFEST_LINK; +use crate::media_signing::{Answered, MediaJob, MediaNamespacing, MediaSigning}; +use crate::origin_auth::OriginSigner; +use crate::platform::test_support::{ + ElapsedTimer, MemoryLoadingCache, StoppedTimer, build_services_with_http_client, +}; +use crate::platform::{ + CacheSpec, PlatformError, PlatformHttpClient, PlatformLoadingCache, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, PlatformTimer, +}; +use crate::test_support::tests::create_test_settings; + +const PATH: &str = "/media/42"; +const FETCHED_URL: &str = "https://origin.example.com/media/42"; +const QUERY_FETCHED_URL: &str = "https://origin.example.com/media/42?v=2"; +const READER_URL: &str = "https://www.example.com/media/42"; +const ORIGIN_IDENTITY: &str = "https://origin.example.com\0origin.example.com"; +const ETAG_V1: &str = "\"v1\""; +const ETAG_V2: &str = "\"v2\""; +/// The size of each chunk the stub origin streams, so every body arrives +/// in several pieces. +const CHUNK: usize = 5; +/// The `Authorization` the test origin signer writes. +const ORIGIN_SIGNATURE: &str = "TestOrigin signed"; + +fn source() -> Vec { + (0_u8..64).collect() +} + +fn other_source() -> Vec { + (100_u8..164).collect() +} + +fn identity() -> VariantIdentity<'static> { + VariantIdentity { + reader_url: READER_URL, + origin_identity: ORIGIN_IDENTITY, + fetched_url: FETCHED_URL, + reader_query_free: true, + } +} + +/// An origin answer for `bytes` at version `etag`: the bytes `range` +/// names as a `206`, or all of them as a `200`, streamed in [`CHUNK`]s and +/// cut with a stream error after `cut_at` bytes where given. +fn serve( + bytes: &[u8], + etag: &str, + range: Option<(u64, u64)>, + cut_at: Option, +) -> Response { + let total = bytes.len(); + let (status, slice) = match range { + Some((first, last)) => { + let first = usize::try_from(first).expect("should fit a test offset"); + let last = usize::try_from(last).expect("should fit a test offset"); + (StatusCode::PARTIAL_CONTENT, bytes[first..=last].to_vec()) + } + None => (StatusCode::OK, bytes.to_vec()), + }; + let mut builder = Response::builder() + .status(status) + .header(header::CONTENT_TYPE, "image/png") + .header(header::ETAG, etag) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header(header::ACCEPT_RANGES, "bytes") + .header(header::CONTENT_LENGTH, slice.len()); + if let Some((first, last)) = range { + builder = builder.header( + header::CONTENT_RANGE, + format!("bytes {first}-{last}/{total}"), + ); + } + let kept = cut_at.map_or(slice.len(), |cut| cut.min(slice.len())); + let mut items: Vec> = slice[..kept] + .chunks(CHUNK) + .map(|chunk| Ok(Bytes::copy_from_slice(chunk))) + .collect(); + if cut_at.is_some() { + items.push(Err(std::io::Error::other("origin connection cut"))); + } + builder + .body(EdgeBody::from_stream(futures::stream::iter(items))) + .expect("should build an origin answer") +} + +/// The origin's whole `200` for the reader's first request. +fn whole() -> Response { + serve(&source(), ETAG_V1, None, None) +} + +/// The origin's answer to a `HEAD`: the whole's headers, no body. +fn head_answer(bytes: &[u8], etag: &str) -> Response { + let (parts, _) = serve(bytes, etag, None, None).into_parts(); + Response::from_parts(parts, EdgeBody::empty()) +} + +fn not_modified(etag: &str) -> Response { + Response::builder() + .status(StatusCode::NOT_MODIFIED) + .header(header::ETAG, etag) + .header(header::CACHE_CONTROL, "public, max-age=600") + .body(EdgeBody::empty()) + .expect("should build a not modified answer") +} + +fn assert_unchanged_not_modified(delivery: Delivery, headers: &HeaderMap) { + let Delivery::Send(response) = delivery else { + panic!("should send the original not modified response"); + }; + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + assert_eq!(response.headers(), headers, "should preserve every header"); + let (_, body) = collect(response); + assert_eq!(body, Some(Vec::new()), "should preserve the empty body"); +} + +/// A `bytes=first-last` request range. +fn requested_range(headers: &HeaderMap) -> Option<(u64, u64)> { + let value = headers.get(header::RANGE)?.to_str().ok()?; + let (first, last) = value.strip_prefix("bytes=")?.split_once('-')?; + Some((first.parse().ok()?, last.parse().ok()?)) +} + +struct VersionChange { + after: usize, + bytes: Vec, + etag: String, + headers: Vec<(HeaderName, HeaderValue)>, +} + +#[derive(Default)] +struct OriginState { + bytes: Vec, + etag: String, + /// From the request after this many on, the origin serves the other + /// version, answering it with these headers too. + flip: Option, + /// Headers every answer carries besides [`serve`]'s own, replacing + /// them. + answer_headers: Vec<(HeaderName, HeaderValue)>, + /// Override the status after this many requests. + status_after: Option<(usize, StatusCode)>, + /// The next answer's stream is cut after this many bytes. + cut_next: Option, + sends: usize, + wrong_content_range: bool, + requests: Vec, +} + +/// An origin that honors `Range` and `If-Match` and can change version. +struct RangeOrigin { + state: Mutex, +} + +impl RangeOrigin { + fn new(bytes: &[u8], etag: &str) -> Arc { + Arc::new(Self { + state: Mutex::new(OriginState { + bytes: bytes.to_vec(), + etag: etag.to_owned(), + ..OriginState::default() + }), + }) + } + + /// Serves `bytes` at `etag` from the request after the `after`th on. + fn flip_after(&self, after: usize, bytes: &[u8], etag: &str) { + self.flip_after_with(after, bytes, etag, Vec::new()); + } + + /// As [`Self::flip_after`], the new version's answers carrying + /// `headers` too. + fn flip_after_with( + &self, + after: usize, + bytes: &[u8], + etag: &str, + headers: Vec<(HeaderName, HeaderValue)>, + ) { + self.state.lock().expect("should lock origin").flip = Some(VersionChange { + after, + bytes: bytes.to_vec(), + etag: etag.to_owned(), + headers, + }); + } + + fn cut_next_at(&self, at: usize) { + self.state.lock().expect("should lock origin").cut_next = Some(at); + } + + fn answer_wrong_content_range(&self) { + self.state + .lock() + .expect("should lock origin") + .wrong_content_range = true; + } + + fn requests(&self) -> Vec { + self.state + .lock() + .expect("should lock origin") + .requests + .clone() + } + + /// The ranges every request asked for, in order. + fn ranges(&self) -> Vec> { + self.requests().iter().map(requested_range).collect() + } + + /// How many requests read the whole version, which only hashing does + /// besides a whole delivery. + fn whole_reads(&self) -> usize { + self.ranges() + .iter() + .filter(|range| **range == Some((0, 63))) + .count() + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformHttpClient for RangeOrigin { + fn supports_streaming_responses(&self) -> bool { + true + } + + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + let mut state = self.state.lock().expect("should lock origin"); + if state + .flip + .as_ref() + .is_some_and(|flip| state.sends >= flip.after) + { + let flip = state.flip.take().expect("should have a due version change"); + state.bytes = flip.bytes; + state.etag = flip.etag; + state.answer_headers = flip.headers; + } + state.sends += 1; + let headers = request.request.headers().clone(); + state.requests.push(headers.clone()); + if let Some(if_match) = headers.get(header::IF_MATCH) + && if_match.as_bytes() != state.etag.as_bytes() + { + let refused = Response::builder() + .status(StatusCode::PRECONDITION_FAILED) + .header(header::ETAG, state.etag.as_str()) + .body(EdgeBody::empty()) + .expect("should build a refusal"); + return Ok(PlatformResponse::new(refused)); + } + let range = requested_range(&headers); + let cut = state.cut_next.take(); + let mut response = serve(&state.bytes, &state.etag, range, cut); + if let Some((after, status)) = state.status_after + && state.sends > after + { + *response.status_mut() = status; + } + for (name, value) in &state.answer_headers { + response.headers_mut().insert(name.clone(), value.clone()); + } + if state.wrong_content_range && range.is_some() { + response.headers_mut().insert( + header::CONTENT_RANGE, + HeaderValue::from_static("bytes 0-0/1"), + ); + } + Ok(PlatformResponse::new(response)) + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } +} + +/// An origin signer that writes [`ORIGIN_SIGNATURE`] and owns every +/// `x-test-origin-*` header. +struct TestOriginSigner; + +impl OriginSigner for TestOriginSigner { + fn sign( + &self, + _method: &Method, + _url: &url::Url, + headers: &mut HeaderMap, + _now: SystemTime, + ) -> Result<(), Report> { + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static(ORIGIN_SIGNATURE), + ); + Ok(()) + } + + fn remove_namespace_headers(&self, headers: &mut HeaderMap) { + headers.remove("x-test-origin-key"); + } +} + +/// One large job the stub signer was handed. +#[derive(Debug, Clone)] +struct LargeJobRecord { + source_sha256: String, + size: u64, + remaining: Duration, + source_reference: Option, + reader_reference: Option, + origin_host: String, +} + +/// How the stub signer answers a large job. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LargeBehavior { + /// Signs, adding [`MANIFEST_LINK`]. + Sign, + /// Signs after confirming the pinned version is still current. + ConfirmThenSign, + /// Refuses, as a source with provenance of its own is refused. + Refuse, + /// Never answers. + Hang, +} + +/// A signer whose bounded limit is 16 bytes and whose large lane takes up +/// to 1 KiB. +struct LargeSigner { + behavior: LargeBehavior, + namespace: Mutex>, + namespace_gate: Mutex>>, + large_duration: Duration, + attempts: Mutex, + jobs: Mutex>, + failures: Mutex>, +} + +impl LargeSigner { + fn new(behavior: LargeBehavior) -> Arc { + Arc::new(Self { + behavior, + namespace: Mutex::new(Ok("policy-1".to_owned())), + namespace_gate: Mutex::new(None), + large_duration: Duration::from_secs(60), + attempts: Mutex::new(0), + jobs: Mutex::new(Vec::new()), + failures: Mutex::new(Vec::new()), + }) + } + + fn set_namespace(&self, namespace: Result) { + *self.namespace.lock().expect("should lock namespace") = namespace; + } + + /// How many times core called `sign_large`. + fn attempts(&self) -> usize { + *self.attempts.lock().expect("should lock attempts") + } + + fn jobs(&self) -> Vec { + self.jobs.lock().expect("should lock jobs").clone() + } + + fn failures(&self) -> Vec<&'static str> { + self.failures.lock().expect("should lock failures").clone() + } +} + +impl MediaSigner for LargeSigner { + fn signer_id(&self) -> &'static str { + "testing.large-media" + } + + fn handles_path(&self, path: &str) -> bool { + !path.starts_with("/api/") + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type.starts_with("image/") + } + + fn max_source_bytes(&self) -> usize { + 16 + } + + fn namespace<'a>(&'a self, _services: &'a RuntimeServices) -> MediaNamespacing<'a> { + let namespace = self + .namespace + .lock() + .expect("should lock namespace") + .clone(); + let gate = self + .namespace_gate + .lock() + .expect("should lock namespace gate") + .take(); + Box::pin(async move { + if let Some(gate) = gate { + gate.await.expect("should release namespace"); + } + namespace + }) + } + + fn sign<'a>( + &'a self, + _services: &'a RuntimeServices, + _job: &'a MediaJob<'a>, + ) -> MediaSigning<'a> { + Box::pin(async { + Err(MediaSigningFailure { + reason: "bounded_not_expected", + }) + }) + } + + fn max_large_source_bytes(&self) -> u64 { + 1024 + } + + fn max_large_media_duration(&self) -> Duration { + self.large_duration + } + + fn sign_large<'a>( + &'a self, + _services: &'a RuntimeServices, + job: &'a LargeMediaJob<'a>, + ) -> LargeMediaSigning<'a> { + Box::pin(async move { + *self.attempts.lock().expect("should lock attempts") += 1; + match self.behavior { + LargeBehavior::Refuse => { + return Err(MediaSigningFailure { + reason: "source_has_embedded_provenance", + }); + } + LargeBehavior::Hang => futures::future::pending::<()>().await, + LargeBehavior::ConfirmThenSign => { + if let Err(error) = job.source.confirm_current().await { + self.failures + .lock() + .expect("should lock failures") + .push(error.code()); + return Err(MediaSigningFailure { + reason: error.code(), + }); + } + } + LargeBehavior::Sign => {} + } + self.jobs + .lock() + .expect("should lock jobs") + .push(LargeJobRecord { + source_sha256: job.source_sha256.to_owned(), + size: job.source.size(), + remaining: job.remaining, + source_reference: job.source_reference.map(str::to_owned), + reader_reference: job.reader_reference.map(str::to_owned), + origin_host: job.source_origin_host.to_owned(), + }); + let namespace = self + .namespace + .lock() + .expect("should lock namespace") + .clone() + .unwrap_or_default(); + Ok(LargeSignedMedia { + headers: vec![(header::LINK, HeaderValue::from_static(MANIFEST_LINK))], + namespace, + }) + }) + } +} + +#[derive(Default)] +struct PendingLargeCache { + calls: Mutex, +} + +#[async_trait::async_trait(?Send)] +impl PlatformLoadingCache for PendingLargeCache { + async fn get_or_load( + &self, + _services: &RuntimeServices, + _spec: &CacheSpec, + _key: &StoreKey, + _load: Loader, + ) -> Result { + *self.calls.lock().expect("should lock cache calls") += 1; + futures::future::pending().await + } + + async fn remove(&self, _services: &RuntimeServices, _spec: &CacheSpec, _key: &StoreKey) { + panic!("should never remove a pending cache outcome"); + } +} + +fn services_with_timer( + signer: &Arc, + origin: &Arc, + timer: Arc, +) -> RuntimeServices { + build_services_with_http_client(Arc::clone(origin) as Arc) + .with_media_signer(Some(Arc::clone(signer) as Arc)) + .with_loading_cache(Arc::new(MemoryLoadingCache::default())) + .with_timer(timer) +} + +fn services_for(signer: &Arc, origin: &Arc) -> RuntimeServices { + services_with_timer(signer, origin, Arc::new(StoppedTimer)) +} + +fn claim_for( + services: &RuntimeServices, + method: &Method, + pairs: &[(HeaderName, &str)], +) -> MediaClaim { + let mut headers = HeaderMap::new(); + headers.insert("sec-fetch-dest", HeaderValue::from_static("image")); + for (name, value) in pairs { + headers.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build a header"), + ); + } + MediaClaim::of_request( + &create_test_settings(), + services, + method, + PATH, + &headers, + true, + ) + .expect("should claim a media path") +} + +/// How core asks the origin again in a test: the address, the request's +/// headers before signing, and the origin's signing where it has one. +struct TestOrigin<'a> { + identity: VariantIdentity<'a>, + headers: HeaderMap, + auth: Option, + unsigned_headers: Vec, + response_header_rewrite: Option>, +} + +impl TestOrigin<'static> { + /// An origin request that forwarded the reader's `pairs`, as a route + /// forwards a reader's range and conditions. + fn forwarding(pairs: &[(HeaderName, &str)]) -> Self { + let mut headers = HeaderMap::new(); + for (name, value) in pairs { + headers.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build a header"), + ); + } + Self { + headers, + ..Self::plain() + } + } + + fn plain() -> Self { + Self { + identity: identity(), + headers: HeaderMap::new(), + auth: None, + unsigned_headers: Vec::new(), + response_header_rewrite: None, + } + } +} + +/// Delivers the origin's `response` through `origin`, by the large lane +/// where core admits it. Whether it was admitted, and the answer. +fn run_through( + claim: &MediaClaim, + services: &RuntimeServices, + origin: &TestOrigin<'_>, + response: Response, +) -> (bool, Response) { + block_on(async { + let facts = OriginFacts::of(response.headers()); + match claim + .deliver( + services, + &origin.identity, + Answered::Origin(facts), + response, + ) + .await + { + Delivery::Large(answer) => { + let url = url::Url::parse(origin.identity.fetched_url) + .expect("should parse the fetched address"); + let pinned = PinnedOrigin { + url: &url, + headers: &origin.headers, + backend_name: "origin", + auth: origin.auth.as_ref(), + unsigned_headers: &origin.unsigned_headers, + response_header_rewrite: origin.response_header_rewrite, + }; + ( + true, + claim + .deliver_large(services, &origin.identity, pinned, answer) + .await, + ) + } + Delivery::Send(response) => (false, response), + Delivery::FetchWhole(_) => { + panic!("should never ask for a bounded whole of a large source") + } + } + }) +} + +fn run( + claim: &MediaClaim, + services: &RuntimeServices, + response: Response, +) -> (bool, Response) { + run_through(claim, services, &TestOrigin::plain(), response) +} + +/// The body's bytes, or `None` where its stream failed. +fn collect(response: Response) -> (Response, Option>) { + let (parts, body) = response.into_parts(); + let bytes = block_on(async { + match body { + EdgeBody::Once(bytes) => Some(bytes.to_vec()), + EdgeBody::Stream(mut stream) => { + let mut collected = Vec::new(); + while let Some(chunk) = stream.next().await { + collected.extend_from_slice(&chunk.ok()?); + } + Some(collected) + } + } + }); + (Response::from_parts(parts, EdgeBody::empty()), bytes) +} + +fn header_of<'a>(response: &'a Response, name: &str) -> Option<&'a str> { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) +} + +fn whole_digest_header(bytes: &[u8]) -> String { + format!( + "sha-256=:{}:", + base64::engine::general_purpose::STANDARD.encode(Sha256::digest(bytes)) + ) +} + +#[test] +fn hashes_a_large_source_once_and_delivers_the_exact_pinned_bytes() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::ConfirmThenSign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + + let (large, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + + assert!( + large, + "should admit a source past the bounded limit with a strong tag" + ); + let jobs = signer.jobs(); + assert_eq!(jobs.len(), 1, "should sign once"); + assert_eq!( + jobs[0].source_sha256, + hex::encode(Sha256::digest(source())), + "should hand the signer the digest of every source byte" + ); + assert_eq!(jobs[0].size, 64, "should pin the declared length"); + assert!( + jobs[0].remaining > Duration::ZERO && jobs[0].remaining <= Duration::from_secs(60), + "should tell the signer what is left of its budget" + ); + assert_eq!(jobs[0].source_reference.as_deref(), Some(FETCHED_URL)); + assert_eq!(jobs[0].reader_reference.as_deref(), Some(READER_URL)); + assert_eq!(jobs[0].origin_host, "origin.example.com"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + body, + Some(source()), + "should deliver the origin's exact bytes" + ); + assert_eq!( + header_of(&response, "etag"), + Some(ETAG_V1), + "should keep the origin's validator" + ); + assert_eq!(header_of(&response, "content-length"), Some("64")); + assert_eq!( + header_of(&response, "link"), + Some(MANIFEST_LINK), + "should carry the manifest link" + ); + assert_eq!( + header_of(&response, "repr-digest"), + Some(whole_digest_header(&source()).as_str()), + "should describe the whole representation" + ); + let requests = origin.requests(); + for request in &requests { + assert_eq!( + request + .get(header::IF_MATCH) + .and_then(|value| value.to_str().ok()), + Some(ETAG_V1), + "should pin every origin read to the version" + ); + assert!( + request.get(header::RANGE).is_some(), + "should ask for exact ranges" + ); + } + assert_eq!( + origin.ranges(), + vec![Some((0, 63)), Some((0, 0)), Some((0, 63))], + "should hash once, confirm, then deliver every byte of the pinned version" + ); +} + +#[test] +fn answers_a_reader_range_with_the_exact_original_bytes() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::ConfirmThenSign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[(header::RANGE, "bytes=10-19")]); + + let (large, response) = run( + &claim, + &services, + serve(&source(), ETAG_V1, Some((10, 19)), None), + ); + let (response, body) = collect(response); + + assert!( + large, + "should admit a part whose total is past the bounded limit" + ); + assert_eq!( + signer.jobs()[0].source_sha256, + hex::encode(Sha256::digest(source())), + "should hash the whole version, not the part" + ); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + header_of(&response, "content-range"), + Some("bytes 10-19/64") + ); + assert_eq!(header_of(&response, "content-length"), Some("10")); + assert_eq!( + body, + Some(source()[10..20].to_vec()), + "should send the original bytes of the range" + ); + assert_eq!(header_of(&response, "link"), Some(MANIFEST_LINK)); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!( + origin.ranges(), + vec![Some((0, 63)), Some((0, 0)), Some((10, 19))], + "should hash the whole pinned version once, then read the reader's range pinned" + ); + + let (_, response) = run( + &claim, + &services, + serve(&source(), ETAG_V1, Some((10, 19)), None), + ); + let (response, body) = collect(response); + assert_eq!( + origin.whole_reads(), + 1, + "a second range should never hash again" + ); + assert_eq!(body, Some(source()[10..20].to_vec())); + assert_eq!(header_of(&response, "link"), Some(MANIFEST_LINK)); +} + +#[test] +fn a_cold_head_starts_no_load_and_a_warm_one_is_decorated() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let head = claim_for(&services, &Method::HEAD, &[]); + + let (large, response) = run(&head, &services, head_answer(&source(), ETAG_V1)); + let (response, body) = collect(response); + assert!(large, "should admit a HEAD of a large source"); + assert!( + origin.requests().is_empty(), + "a cold HEAD should never read the origin again" + ); + assert_eq!( + signer.attempts(), + 0, + "a cold HEAD should never call the signer" + ); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(header_of(&response, "content-length"), Some("64")); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!(header_of(&response, "link"), None); + assert_eq!(body, Some(Vec::new()), "a HEAD carries no body"); + + let get = claim_for(&services, &Method::GET, &[]); + let _ = collect(run(&get, &services, whole()).1); + let reads = origin.requests().len(); + let (_, response) = run(&head, &services, head_answer(&source(), ETAG_V1)); + let (response, body) = collect(response); + assert_eq!( + header_of(&response, "link"), + Some(MANIFEST_LINK), + "a warm HEAD carries the manifest" + ); + assert_eq!( + header_of(&response, "repr-digest"), + Some(whole_digest_header(&source()).as_str()) + ); + assert_eq!(header_of(&response, "content-length"), Some("64")); + assert_eq!(body, Some(Vec::new())); + assert_eq!( + origin.requests().len(), + reads, + "a warm HEAD should read nothing" + ); + assert_eq!(signer.attempts(), 1); +} + +#[test] +fn only_an_origin_that_says_it_serves_byte_ranges_starts_a_load() { + for accept_ranges in [None, Some("none")] { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let mut answer = whole(); + answer.headers_mut().remove(header::ACCEPT_RANGES); + if let Some(value) = accept_ranges { + answer + .headers_mut() + .insert(header::ACCEPT_RANGES, HeaderValue::from_static(value)); + } + + let (_, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + + assert!( + origin.requests().is_empty(), + "{accept_ranges:?}: should never hash" + ); + assert_eq!(signer.attempts(), 0, "{accept_ranges:?}"); + assert_eq!( + body, + Some(source()), + "{accept_ranges:?}: should send the origin's answer" + ); + assert_eq!( + header_of(&response, "etag"), + Some(ETAG_V1), + "{accept_ranges:?}" + ); + assert_eq!(header_of(&response, "link"), None, "{accept_ranges:?}"); + } + + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let _ = collect(run(&claim, &services, whole()).1); + let mut answer = whole(); + answer.headers_mut().remove(header::ACCEPT_RANGES); + let (_, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + assert_eq!(body, Some(source())); + assert_eq!( + header_of(&response, "link"), + Some(MANIFEST_LINK), + "a kept signed outcome still decorates the version" + ); + assert_eq!(signer.attempts(), 1); +} + +#[test] +fn a_kept_outcome_answers_without_hashing_and_never_outlives_the_policy() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let _ = collect(run(&claim, &services, whole()).1); + let reads = origin.requests().len(); + + let (_, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + assert_eq!( + signer.attempts(), + 1, + "should reuse the outcome kept for the version" + ); + assert_eq!( + origin.requests().len(), + reads, + "should send the origin's own answer" + ); + assert_eq!(body, Some(source())); + assert_eq!(header_of(&response, "link"), Some(MANIFEST_LINK)); + + let (_, revalidated) = run(&claim, &services, not_modified(ETAG_V1)); + assert_eq!(revalidated.status(), StatusCode::NOT_MODIFIED); + assert_eq!( + header_of(&revalidated, "link"), + Some(MANIFEST_LINK), + "should tell a revalidating reader of the manifest" + ); + + signer.set_namespace(Ok("policy-2".to_owned())); + let _ = collect(run(&claim, &services, whole()).1); + assert_eq!( + signer.attempts(), + 2, + "a changed policy should never find the old outcome" + ); + + signer.set_namespace(Err(MediaSigningFailure { reason: "revoked" })); + let reads = origin.requests().len(); + let (_, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + assert_eq!(signer.attempts(), 2, "a revoked policy should sign nothing"); + assert_eq!(origin.requests().len(), reads, "should hash nothing"); + assert_eq!( + header_of(&response, "link"), + None, + "should carry no manifest" + ); + assert_eq!( + header_of(&response, "etag"), + Some(ETAG_V1), + "should keep the origin's validator on its unchanged bytes" + ); + assert_eq!(body, Some(source())); +} + +#[test] +fn a_refused_source_is_hashed_once_and_then_sent_as_it_came() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[(header::RANGE, "bytes=0-9")]); + let forwarding = TestOrigin::forwarding(&[(header::RANGE, "bytes=0-9")]); + + let (_, response) = run_through( + &claim, + &services, + &forwarding, + serve(&source(), ETAG_V1, Some((0, 9)), None), + ); + let (response, body) = collect(response); + assert_eq!(signer.attempts(), 1); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(header_of(&response, "content-range"), Some("bytes 0-9/64")); + assert_eq!(body, Some(source()[0..10].to_vec())); + assert_eq!(header_of(&response, "link"), None); + assert_eq!( + header_of(&response, "etag"), + Some(ETAG_V1), + "should keep the origin's validator" + ); + assert_eq!( + header_of(&response, "cache-control"), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "a fresh fallback is held by no cache, even for the same version" + ); + let fresh = origin + .requests() + .last() + .cloned() + .expect("should ask the origin afresh"); + assert_eq!( + requested_range(&fresh), + Some((0, 9)), + "should ask afresh exactly as the reader asked" + ); + assert!( + fresh.get(header::IF_MATCH).is_none(), + "an unsigned answer is not pinned" + ); + let reads = origin.requests().len(); + + for _ in 0..3 { + let (_, response) = run_through( + &claim, + &services, + &forwarding, + serve(&source(), ETAG_V1, Some((0, 9)), None), + ); + let (response, body) = collect(response); + assert_eq!(body, Some(source()[0..10].to_vec())); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!(header_of(&response, "link"), None); + } + assert_eq!( + signer.attempts(), + 1, + "a kept refusal should never call the signer again" + ); + assert_eq!( + origin.requests().len(), + reads, + "a kept refusal should never read the origin again" + ); + assert_eq!(origin.whole_reads(), 1, "should hash once in all"); +} + +#[test] +fn a_declined_signed_origin_is_asked_afresh_with_a_fresh_signature() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[(header::RANGE, "bytes=0-9")]); + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); + headers.insert( + header::REFERER, + HeaderValue::from_static("https://www.example.com/page"), + ); + headers.insert("x-test-origin-key", HeaderValue::from_static("from-reader")); + let signed = TestOrigin { + headers, + auth: Some(OriginSigning::new(Arc::new(TestOriginSigner), None)), + unsigned_headers: vec![header::REFERER], + ..TestOrigin::plain() + }; + + let (_, response) = run_through( + &claim, + &services, + &signed, + serve(&source(), ETAG_V1, Some((0, 9)), None), + ); + let (response, body) = collect(response); + + assert_eq!(body, Some(source()[0..10].to_vec())); + assert_eq!(header_of(&response, "link"), None); + let fresh = origin + .requests() + .last() + .cloned() + .expect("should ask the origin afresh"); + assert_eq!( + fresh + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some(ORIGIN_SIGNATURE), + "should sign the fresh request afresh" + ); + assert_eq!(requested_range(&fresh), Some((0, 9))); + assert!(fresh.get(header::REFERER).is_none()); + assert!(fresh.get("x-test-origin-key").is_none()); + assert!(fresh.get(header::IF_MATCH).is_none()); +} + +#[test] +fn a_not_modified_pending_namespace_obeys_the_large_budget() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let mut signer = LargeSigner::new(LargeBehavior::Sign); + let (namespace_ready, namespace_gate) = oneshot::channel(); + let configured = Arc::get_mut(&mut signer).expect("should exclusively own the signer"); + configured.large_duration = Duration::from_secs(5); + *configured + .namespace_gate + .lock() + .expect("should lock namespace gate") = Some(namespace_gate); + let timer = Arc::new(ElapsedTimer::default()); + let cache = Arc::new(PendingLargeCache::default()); + let services = services_with_timer( + &signer, + &origin, + Arc::clone(&timer) as Arc, + ) + .with_loading_cache(Arc::clone(&cache) as Arc); + let claim = claim_for(&services, &Method::GET, &[]); + let response = not_modified(ETAG_V1); + let headers = response.headers().clone(); + let facts = OriginFacts::of(response.headers()); + + let delivery = claim + .deliver(&services, &identity(), Answered::Origin(facts), response) + .now_or_never() + .expect("should stop waiting for the namespace at the large deadline"); + + assert_unchanged_not_modified(delivery, &headers); + assert_eq!(*cache.calls.lock().expect("should lock cache calls"), 0); + assert!(origin.requests().is_empty()); + assert_eq!(signer.attempts(), 0); + let requested = timer.requested.lock().expect("should lock timer requests"); + assert_eq!(requested.len(), 1); + assert!(requested[0] > Duration::ZERO && requested[0] <= Duration::from_secs(5)); + drop(namespace_ready); +} + +#[test] +fn a_not_modified_pending_cache_obeys_the_large_budget() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let mut signer = LargeSigner::new(LargeBehavior::Sign); + Arc::get_mut(&mut signer) + .expect("should exclusively own the signer") + .large_duration = Duration::from_secs(5); + let timer = Arc::new(ElapsedTimer::default()); + let cache = Arc::new(PendingLargeCache::default()); + let services = services_with_timer( + &signer, + &origin, + Arc::clone(&timer) as Arc, + ) + .with_loading_cache(Arc::clone(&cache) as Arc); + let claim = claim_for(&services, &Method::GET, &[]); + let response = not_modified(ETAG_V1); + let headers = response.headers().clone(); + let facts = OriginFacts::of(response.headers()); + + let delivery = claim + .deliver(&services, &identity(), Answered::Origin(facts), response) + .now_or_never() + .expect("should stop waiting for the cached outcome at the large deadline"); + + assert_unchanged_not_modified(delivery, &headers); + assert_eq!(*cache.calls.lock().expect("should lock cache calls"), 1); + assert!(origin.requests().is_empty()); + assert_eq!(signer.attempts(), 0); + let requested = timer.requested.lock().expect("should lock timer requests"); + assert_eq!(requested.len(), 1); + assert!(requested[0] > Duration::ZERO && requested[0] <= Duration::from_secs(5)); +} + +#[test] +fn a_not_modified_without_a_timer_or_positive_large_budget_starts_no_io() { + for (has_timer, budget) in [(false, Duration::from_secs(5)), (true, Duration::ZERO)] { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let mut signer = LargeSigner::new(LargeBehavior::Sign); + let (namespace_ready, namespace_gate) = oneshot::channel(); + let configured = Arc::get_mut(&mut signer).expect("should exclusively own the signer"); + configured.large_duration = budget; + *configured + .namespace_gate + .lock() + .expect("should lock namespace gate") = Some(namespace_gate); + let cache = Arc::new(PendingLargeCache::default()); + let services = + build_services_with_http_client(Arc::clone(&origin) as Arc) + .with_media_signer(Some(Arc::clone(&signer) as Arc)) + .with_loading_cache(Arc::clone(&cache) as Arc); + let services = if has_timer { + services.with_timer(Arc::new(ElapsedTimer::default())) + } else { + services + }; + let claim = claim_for(&services, &Method::GET, &[]); + let response = not_modified(ETAG_V1); + let headers = response.headers().clone(); + let facts = OriginFacts::of(response.headers()); + + let delivery = claim + .deliver(&services, &identity(), Answered::Origin(facts), response) + .now_or_never() + .expect("should refuse before polling namespace without a timer and positive budget"); + + assert_unchanged_not_modified(delivery, &headers); + assert!( + signer + .namespace_gate + .lock() + .expect("should lock namespace gate") + .is_some(), + "should never call namespace resolution" + ); + assert_eq!(*cache.calls.lock().expect("should lock cache calls"), 0); + assert!(origin.requests().is_empty()); + assert_eq!(signer.attempts(), 0); + drop(namespace_ready); + } +} + +#[test] +fn a_namespace_pending_past_the_large_budget_starts_no_hash_or_signing() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let mut signer = LargeSigner::new(LargeBehavior::Sign); + let (namespace_ready, namespace_gate) = oneshot::channel(); + let configured = Arc::get_mut(&mut signer).expect("should exclusively own the signer"); + configured.large_duration = Duration::from_secs(5); + *configured + .namespace_gate + .lock() + .expect("should lock namespace gate") = Some(namespace_gate); + let timer = Arc::new(ElapsedTimer::default()); + let services = services_with_timer( + &signer, + &origin, + Arc::clone(&timer) as Arc, + ); + let claim = claim_for(&services, &Method::GET, &[]); + let response = whole(); + let facts = OriginFacts::of(response.headers()); + let Delivery::Large(answer) = + block_on(claim.deliver(&services, &identity(), Answered::Origin(facts), response)) + else { + panic!("should admit the source to the large lane"); + }; + let url = url::Url::parse(FETCHED_URL).expect("should parse the fetched address"); + let headers = HeaderMap::new(); + let pinned = PinnedOrigin { + url: &url, + headers: &headers, + backend_name: "origin", + auth: None, + unsigned_headers: &[], + response_header_rewrite: None, + }; + + // Keep the policy future unresolved while the deterministic large timer + // expires. A missing namespace race must fail here, never hang the test. + let response = claim + .deliver_large(&services, &identity(), pinned, answer) + .now_or_never() + .expect("should return unsigned when the large budget expires during namespace resolution"); + let (response, body) = collect(response); + + assert_eq!(body, Some(source()), "should preserve the original body"); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!(header_of(&response, "link"), None); + assert_eq!(header_of(&response, "repr-digest"), None); + assert!(origin.requests().is_empty(), "should never start hashing"); + assert_eq!(signer.attempts(), 0, "should never start signing"); + assert!(signer.jobs().is_empty()); + let requested = timer.requested.lock().expect("should lock timer requests"); + assert_eq!(requested.len(), 1, "should race the namespace once"); + assert!( + requested[0] > Duration::ZERO && requested[0] <= Duration::from_secs(5), + "should use the large budget rather than the policy's own hold limit" + ); + drop(namespace_ready); +} + +#[test] +fn an_exhausted_large_budget_starts_no_hash_or_signing() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_with_timer(&signer, &origin, Arc::new(ElapsedTimer::default())); + let load = LargeLoad { + signer: Arc::clone(&signer) as Arc, + services, + url: url::Url::parse(FETCHED_URL).expect("should parse the fetched address"), + headers: HeaderMap::new(), + backend_name: "origin".to_owned(), + auth: None, + unsigned_headers: Vec::new(), + version: SourceVersion::new(ETAG_V1, to_u64(source().len())) + .expect("should pin the test version"), + reader_url: READER_URL.to_owned(), + content_type: "image/png".to_owned(), + references: LargeReferences::of(&identity(), false), + namespace: "policy-1".to_owned(), + budget: LargeBudget { + started: Instant::now(), + total: Duration::ZERO, + }, + keep_for: Duration::from_secs(600), + }; + + let loaded = block_on(load.run()); + + assert!( + matches!( + decode_kept_outcome(&loaded.bytes), + Some(KeptOutcome::Declined) + ), + "should keep an unsigned outcome" + ); + assert!(origin.requests().is_empty(), "should never start hashing"); + assert_eq!(signer.attempts(), 0, "should never start signing"); +} + +#[test] +fn a_timed_out_source_is_hashed_once_and_then_sent_as_it_came() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Hang); + let services = services_with_timer(&signer, &origin, Arc::new(ElapsedTimer::default())); + let claim = claim_for(&services, &Method::GET, &[]); + + let (large, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + assert!(large); + assert!( + signer.jobs().is_empty(), + "a signer out of time signs nothing" + ); + assert_eq!(body, Some(source())); + assert_eq!(header_of(&response, "link"), None); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + let reads = origin.requests().len(); + + let (_, response) = run(&claim, &services, whole()); + let (_, body) = collect(response); + assert_eq!(body, Some(source())); + assert_eq!( + signer.attempts(), + 1, + "a kept timeout should never call the signer again" + ); + assert_eq!( + origin.requests().len(), + reads, + "a kept timeout should never hash again" + ); +} + +#[test] +fn a_version_change_before_publication_signs_nothing_and_sends_the_current_version() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + // The hash reads the first version; the signer's confirmation and the + // fresh answer meet the second. + origin.flip_after(1, &other_source(), ETAG_V2); + let signer = LargeSigner::new(LargeBehavior::ConfirmThenSign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + + let (large, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + + assert!(large); + assert!(signer.jobs().is_empty(), "should publish nothing"); + assert_eq!(signer.failures(), vec!["source_version_changed"]); + assert_eq!( + response.status(), + StatusCode::OK, + "an unsigned answer is the origin's, not a refusal" + ); + assert_eq!( + body, + Some(other_source()), + "should send the version the origin now serves" + ); + assert_eq!( + header_of(&response, "etag"), + Some(ETAG_V2), + "should describe the bytes sent by their own validator" + ); + assert_eq!(header_of(&response, "link"), None); + assert_eq!(header_of(&response, "repr-digest"), None); +} + +#[test] +fn a_fresh_fallback_is_never_cached_and_keeps_both_security_policies() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let caller_policy = "img-src 'self'"; + let fresh_policy = "default-src 'none'"; + // The same version answers afresh with a shorter lifetime and a + // security policy of its own. + origin.flip_after_with( + 1, + &source(), + ETAG_V1, + vec![ + ( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=5"), + ), + ( + header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static(fresh_policy), + ), + ], + ); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let mut answer = whole(); + answer.headers_mut().insert( + header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static(caller_policy), + ); + + let (_, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + + assert_eq!(body, Some(source())); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!( + header_of(&response, "cache-control"), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "should keep neither the first answer's lifetime nor the fresh one's" + ); + let policies: Vec<_> = response + .headers() + .get_all(header::CONTENT_SECURITY_POLICY) + .iter() + .collect(); + assert_eq!( + policies, + vec![caller_policy, fresh_policy], + "should add the fresh policy to the caller's, never replace it" + ); +} + +#[test] +fn a_refused_source_that_changed_keeps_the_new_versions_provenance_and_privacy() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let origin_link = "; rel=\"c2pa-manifest\""; + // The hash reads the first version; the fresh answer meets a second + // that names a manifest of its own and may be held by no shared cache. + origin.flip_after_with( + 1, + &other_source(), + ETAG_V2, + vec![ + ( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ), + (header::LINK, HeaderValue::from_static(origin_link)), + ( + header::SET_COOKIE, + HeaderValue::from_static("session=reader-1"), + ), + ], + ); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let mut answer = whole(); + answer.headers_mut().insert( + header::EXPIRES, + HeaderValue::from_static("Wed, 21 Oct 2026 07:28:00 GMT"), + ); + answer + .headers_mut() + .insert(header::AGE, HeaderValue::from_static("30")); + + let (_, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + + assert_eq!(signer.attempts(), 1); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + body, + Some(other_source()), + "should send the version the origin now serves" + ); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V2)); + let links: Vec<_> = response.headers().get_all(header::LINK).iter().collect(); + assert_eq!( + links, + vec![origin_link], + "should keep the new version's manifest link" + ); + assert_eq!( + header_of(&response, "cache-control"), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "should never send the first version's shared cache policy with other bytes" + ); + assert_eq!(header_of(&response, "expires"), None); + assert_eq!(header_of(&response, "age"), None); + assert_eq!( + header_of(&response, "set-cookie"), + None, + "should pass on no cookie the caller's handling never saw" + ); + assert_eq!(header_of(&response, "repr-digest"), None); +} + +#[test] +fn a_fresh_fallback_keeps_redirect_challenge_and_retry_response_semantics() { + for (status, name, value) in [ + ( + StatusCode::TEMPORARY_REDIRECT, + header::LOCATION, + "/media/moved", + ), + ( + StatusCode::UNAUTHORIZED, + header::WWW_AUTHENTICATE, + "Bearer realm=\"media\"", + ), + (StatusCode::SERVICE_UNAVAILABLE, header::RETRY_AFTER, "60"), + ] { + let origin = RangeOrigin::new(&source(), ETAG_V1); + origin.flip_after_with( + 1, + &source(), + ETAG_V1, + vec![(name.clone(), HeaderValue::from_static(value))], + ); + origin + .state + .lock() + .expect("should lock origin") + .status_after = Some((1, status)); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let (_, response) = run(&claim, &services, whole()); + let (response, _) = collect(response); + + assert_eq!(response.status(), status); + assert_eq!( + response.headers().get(&name), + Some(&HeaderValue::from_static(value)) + ); + assert_eq!( + header_of(&response, "cache-control"), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + ); + assert_eq!(signer.attempts(), 1); + assert_eq!( + origin.requests().len(), + 2, + "should not follow or retry the fresh response" + ); + } +} + +#[test] +fn a_fresh_publisher_redirect_uses_the_existing_origin_url_mapping_without_discarded_cookies() { + for (location, expected) in [ + ( + "https://origin.example.com/media/moved", + "https://www.example.com/media/moved", + ), + (FETCHED_URL, FETCHED_URL), + ] { + let origin = RangeOrigin::new(&source(), ETAG_V1); + origin.flip_after_with( + 1, + &source(), + ETAG_V1, + vec![ + (header::LOCATION, HeaderValue::from_static(location)), + ( + header::SET_COOKIE, + HeaderValue::from_static("session=new; Path=/"), + ), + ( + header::CONTENT_LOCATION, + HeaderValue::from_static("https://origin.example.com/media/fresh"), + ), + ], + ); + origin + .state + .lock() + .expect("should lock origin") + .status_after = Some((1, StatusCode::TEMPORARY_REDIRECT)); + let signer = LargeSigner::new(LargeBehavior::Refuse); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let publisher = TestOrigin { + response_header_rewrite: Some(crate::response_header_rewrite::OriginHeaderRewrite { + origin_host: "origin.example.com", + origin_host_header: "origin.example.com", + origin_path: "", + request_host: "www.example.com", + request_scheme: "https", + origin_scheme: "https", + request_path_and_query: PATH, + request_is_get_or_head: true, + }), + ..TestOrigin::plain() + }; + let mut initial = whole(); + initial + .headers_mut() + .insert(header::CONTENT_LOCATION, HeaderValue::from_static("/stale")); + let (_, response) = run_through(&claim, &services, &publisher, initial); + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + assert_eq!( + response.headers().get(header::LOCATION), + Some(&HeaderValue::from_static(expected)), + "a discarded cookie must not justify redirecting to the same serving URL", + ); + assert!(!response.headers().contains_key(header::SET_COOKIE)); + assert_eq!( + response.headers().get(header::CONTENT_LOCATION), + Some(&HeaderValue::from_static( + "https://www.example.com/media/fresh" + )), + ); + } +} + +#[test] +fn a_version_change_after_signing_is_refused_rather_than_mixed() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + // The hash reads the first version; the signed delivery meets the + // second. + origin.flip_after(1, &other_source(), ETAG_V2); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + + let (_, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + + assert_eq!(signer.jobs().len(), 1); + assert_eq!( + response.status(), + StatusCode::SERVICE_UNAVAILABLE, + "should never send other bytes under the signed version's headers" + ); + assert_eq!(header_of(&response, "link"), None); + assert_eq!(body, Some(Vec::new())); +} + +#[test] +fn a_cut_hash_read_signs_nothing_and_is_never_resumed() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + origin.cut_next_at(23); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + + let (large, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + + assert!(large); + assert_eq!(signer.attempts(), 0, "should sign nothing from a cut read"); + assert_eq!( + origin.ranges(), + vec![Some((0, 63)), None], + "should neither resume nor restart the hash, only ask the origin afresh once as the reader asked" + ); + assert_eq!( + body, + Some(source()), + "should deliver the origin's fresh answer" + ); + assert_eq!( + header_of(&response, "link"), + None, + "should carry no manifest" + ); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + + let (_, response) = run(&claim, &services, whole()); + let (_, body) = collect(response); + assert_eq!(body, Some(source())); + assert_eq!( + origin.requests().len(), + 2, + "a kept failure should never hash again" + ); + + let origin = RangeOrigin::new(&source(), ETAG_V1); + origin.cut_next_at(23); + origin.flip_after(1, &other_source(), ETAG_V2); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let (_, response) = run(&claim, &services, whole()); + let (response, body) = collect(response); + assert_eq!(signer.attempts(), 0); + assert_eq!( + response.status(), + StatusCode::OK, + "an unsigned answer is the origin's, not a refusal" + ); + assert_eq!(body, Some(other_source())); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V2)); + assert_eq!(header_of(&response, "link"), None); +} + +#[test] +fn a_source_no_public_address_names_is_never_hashed() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let queried = TestOrigin { + identity: VariantIdentity { + fetched_url: QUERY_FETCHED_URL, + reader_query_free: false, + ..identity() + }, + ..TestOrigin::plain() + }; + + for _ in 0..2 { + let (_, response) = run_through(&claim, &services, &queried, whole()); + let (response, body) = collect(response); + assert_eq!( + body, + Some(source()), + "should send the origin's answer as it came" + ); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!(header_of(&response, "link"), None); + } + assert!( + origin.requests().is_empty(), + "should never read a source no public address names" + ); + assert_eq!(signer.attempts(), 0); + + let reader_named = TestOrigin { + identity: VariantIdentity { + fetched_url: QUERY_FETCHED_URL, + ..identity() + }, + ..TestOrigin::plain() + }; + let _ = collect(run_through(&claim, &services, &reader_named, whole()).1); + let jobs = signer.jobs(); + assert_eq!( + jobs[0].source_reference, None, + "should never name a fetched address with a query" + ); + assert_eq!( + jobs[0].reader_reference.as_deref(), + Some(READER_URL), + "should name the reader's query-free address" + ); +} + +#[test] +fn a_signed_origin_address_is_never_named_and_its_reads_carry_no_reader_header() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::ConfirmThenSign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let mut headers = HeaderMap::new(); + headers.insert(header::HOST, HeaderValue::from_static("origin.example.com")); + headers.insert( + header::REFERER, + HeaderValue::from_static("https://www.example.com/page"), + ); + headers.insert( + header::IF_NONE_MATCH, + HeaderValue::from_static("\"reader\""), + ); + headers.insert("x-test-origin-key", HeaderValue::from_static("from-reader")); + let signed = TestOrigin { + headers, + auth: Some(OriginSigning::new(Arc::new(TestOriginSigner), None)), + unsigned_headers: vec![header::REFERER], + ..TestOrigin::plain() + }; + + let (_, response) = run_through(&claim, &services, &signed, whole()); + let (response, body) = collect(response); + + let jobs = signer.jobs(); + assert_eq!( + jobs[0].source_reference, None, + "should never name an address core signs" + ); + assert_eq!(jobs[0].reader_reference.as_deref(), Some(READER_URL)); + assert_eq!(body, Some(source())); + assert_eq!(header_of(&response, "link"), Some(MANIFEST_LINK)); + let requests = origin.requests(); + assert_eq!(requests.len(), 3, "should hash, confirm and deliver"); + for request in &requests { + assert_eq!( + request + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + Some(ORIGIN_SIGNATURE), + "should sign every pinned read afresh" + ); + assert!( + request.get(header::REFERER).is_none(), + "should send no reader header a signed read must not carry" + ); + assert!( + request.get(header::IF_NONE_MATCH).is_none(), + "should send none of the reader's conditions" + ); + assert!( + request.get("x-test-origin-key").is_none(), + "should send no header in the signer's namespace" + ); + assert_eq!( + request + .get(header::IF_MATCH) + .and_then(|value| value.to_str().ok()), + Some(ETAG_V1) + ); + } + + let queried_reader = TestOrigin { + identity: VariantIdentity { + reader_query_free: false, + ..identity() + }, + auth: Some(OriginSigning::new(Arc::new(TestOriginSigner), None)), + ..TestOrigin::plain() + }; + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let _ = collect(run_through(&claim, &services, &queried_reader, whole()).1); + assert!( + origin.requests().is_empty(), + "should decline before hashing when only the signed address could name the source" + ); + assert_eq!(signer.attempts(), 0); +} + +#[test] +fn an_answer_whose_outcome_cannot_be_kept_is_never_hashed() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + for cache_control in ["no-cache", "public"] { + let mut answer = whole(); + answer.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static(cache_control), + ); + let (large, response) = run(&claim, &services, answer); + let (_, body) = collect(response); + assert!(!large, "{cache_control}: should not enter the large lane"); + assert_eq!(body, Some(source())); + } + + let keeps_nothing = + build_services_with_http_client(Arc::clone(&origin) as Arc) + .with_media_signer(Some(Arc::clone(&signer) as Arc)) + .with_timer(Arc::new(StoppedTimer)); + let claim = claim_for(&keeps_nothing, &Method::GET, &[]); + let (large, response) = run(&claim, &keeps_nothing, whole()); + let (_, body) = collect(response); + assert!( + !large, + "should not enter the large lane where nothing is kept" + ); + assert_eq!(body, Some(source())); + + assert!( + origin.requests().is_empty(), + "should never read the origin again" + ); + assert_eq!(signer.attempts(), 0); +} + +#[test] +fn an_origin_manifest_link_is_kept_rather_than_competed_with() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let origin_link = "; rel=\"c2pa-manifest\""; + let mut answer = whole(); + answer + .headers_mut() + .insert(header::LINK, HeaderValue::from_static(origin_link)); + + let (large, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + + assert!( + !large, + "should not sign a source that names its own manifest" + ); + assert_eq!(body, Some(source())); + let links: Vec<_> = response.headers().get_all(header::LINK).iter().collect(); + assert_eq!( + links, + vec![origin_link], + "should keep only the origin's manifest link" + ); + assert!(origin.requests().is_empty()); + assert_eq!(signer.attempts(), 0); +} + +#[test] +fn a_warm_not_modified_naming_an_origin_manifest_gets_no_kept_outcome() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let _ = collect(run(&claim, &services, whole()).1); + assert_eq!( + signer.jobs().len(), + 1, + "should keep a signed outcome for the version" + ); + + let (_, plain) = run(&claim, &services, not_modified(ETAG_V1)); + assert_eq!( + header_of(&plain, "link"), + Some(MANIFEST_LINK), + "the kept outcome decorates a 304 without a link of the origin's" + ); + + let origin_link = "; rel=\"c2pa-manifest\""; + let mut linked = not_modified(ETAG_V1); + linked + .headers_mut() + .insert(header::LINK, HeaderValue::from_static(origin_link)); + let (_, response) = run(&claim, &services, linked); + + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + let links: Vec<_> = response.headers().get_all(header::LINK).iter().collect(); + assert_eq!( + links, + vec![origin_link], + "should keep the origin's manifest link alone" + ); + assert_eq!(header_of(&response, "repr-digest"), None); + assert_eq!(header_of(&response, "etag"), Some(ETAG_V1)); + assert_eq!(signer.attempts(), 1, "should never sign again"); +} + +#[test] +fn a_source_without_a_strong_tag_is_never_given_a_version() { + for etag in ["W/\"v1\"", ""] { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let claim = claim_for(&services, &Method::GET, &[]); + let mut answer = whole(); + if etag.is_empty() { + answer.headers_mut().remove(header::ETAG); + } else { + answer + .headers_mut() + .insert(header::ETAG, HeaderValue::from_static("W/\"v1\"")); + } + + let (large, response) = run(&claim, &services, answer); + let (response, body) = collect(response); + + assert!(!large, "{etag:?}: should not enter the large lane"); + assert_eq!(signer.attempts(), 0); + assert!( + origin.requests().is_empty(), + "{etag:?}: should not ask the origin again" + ); + assert_eq!( + body, + Some(source()), + "{etag:?}: should send the origin's answer as it came" + ); + assert_eq!(header_of(&response, "link"), None); + } +} + +#[test] +fn a_pinned_read_refuses_another_version_or_range() { + let origin = RangeOrigin::new(&source(), ETAG_V1); + let signer = LargeSigner::new(LargeBehavior::Sign); + let services = services_for(&signer, &origin); + let url = url::Url::parse(FETCHED_URL).expect("should parse the fetched address"); + let headers = HeaderMap::new(); + let version = SourceVersion::new(ETAG_V1, 64).expect("should take a strong tag"); + let reader = VersionedMediaSource::new(&services, &url, &headers, "origin", version); + + let mut part = [0_u8; 8]; + block_on(reader.read_exact_at(8, &mut part)).expect("should read the pinned range"); + assert_eq!(part.to_vec(), source()[8..16].to_vec()); + assert_eq!( + block_on(reader.read_exact_at(60, &mut part)), + Err(MediaSourceError::OutOfBounds), + "should refuse a read past the version before asking" + ); + + origin.answer_wrong_content_range(); + assert_eq!( + block_on(reader.read_exact_at(8, &mut part)), + Err(MediaSourceError::InvalidAnswer { + reason: "pinned_read_content_range" + }) + ); + + origin.flip_after(0, &other_source(), ETAG_V2); + assert_eq!( + block_on(reader.confirm_current()), + Err(MediaSourceError::VersionChanged) + ); + assert!( + SourceVersion::new("W/\"v1\"", 64).is_none(), + "a weak tag names no version" + ); +} diff --git a/crates/trusted-server-core/src/media_signing/test_support.rs b/crates/trusted-server-core/src/media_signing/test_support.rs new file mode 100644 index 0000000000..f6e5ddb075 --- /dev/null +++ b/crates/trusted-server-core/src/media_signing/test_support.rs @@ -0,0 +1,164 @@ +//! A [`MediaSigner`] for core's tests, which records each job and appends a +//! fixed suffix to the source in place of a manifest. + +use std::sync::{Arc, Mutex}; + +use bytes::Bytes; +use http::{HeaderName, HeaderValue, header}; + +use super::{ + MediaJob, MediaNamespacing, MediaSigner, MediaSigning, MediaSigningFailure, SignedMedia, +}; +use crate::platform::RuntimeServices; + +/// What the stub's signing appends to the source. +pub(crate) const SIGNED_SUFFIX: &[u8] = b"|signed-manifest"; + +/// The header the stub writes on a signed representation. +pub(crate) const MANIFEST_HEADER: &str = "x-test-manifest"; + +/// The `Link` the stub's [`Behavior::SignWithLink`] writes. +pub(crate) const MANIFEST_LINK: &str = "; rel=\"c2pa-manifest\""; + +/// How the stub answers a job. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Behavior { + /// Returns the source with [`SIGNED_SUFFIX`] appended. + Sign, + /// Returns a failure. + Fail, + /// Returns signed bytes with a header core owns. + ReservedHeader, + /// Signs as [`Behavior::Sign`] and adds a manifest [`MANIFEST_LINK`]. + SignWithLink, +} + +/// One job the stub was handed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RecordedJob { + pub(crate) reader_url: String, + pub(crate) content_type: String, + pub(crate) source: Vec, + pub(crate) source_sha256: String, + pub(crate) source_reference: Option, + pub(crate) source_origin_host: Option, + pub(crate) optimizer_output: bool, +} + +/// Leaves `/api/` paths alone and signs every `image/` type. +pub(crate) struct StubSigner { + behavior: Behavior, + limit: usize, + jobs: Mutex>, + namespace: Mutex>, +} + +impl StubSigner { + pub(crate) fn new(behavior: Behavior, limit: usize) -> Arc { + Arc::new(Self { + behavior, + limit, + jobs: Mutex::new(Vec::new()), + namespace: Mutex::new(Ok("policy-1".to_owned())), + }) + } + + /// Makes [`MediaSigner::namespace`] answer `namespace` from now on, as a + /// changed or revoked policy would. + pub(crate) fn set_namespace(&self, namespace: Result) { + *self.namespace.lock().expect("should lock namespace") = namespace; + } + + pub(crate) fn jobs(&self) -> Vec { + self.jobs.lock().expect("should lock jobs").clone() + } + + /// `services` carrying this signer, as an adapter hands it in. + pub(crate) fn installed_in(self: &Arc, services: RuntimeServices) -> RuntimeServices { + services.with_media_signer(Some(Arc::clone(self) as Arc)) + } +} + +impl MediaSigner for StubSigner { + fn signer_id(&self) -> &'static str { + "testing.media" + } + + fn handles_path(&self, path: &str) -> bool { + !path.starts_with("/api/") + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type.starts_with("image/") + } + + fn max_source_bytes(&self) -> usize { + self.limit + } + + fn namespace<'a>(&'a self, _services: &'a RuntimeServices) -> MediaNamespacing<'a> { + let namespace = self + .namespace + .lock() + .expect("should lock namespace") + .clone(); + Box::pin(async move { namespace }) + } + + fn sign<'a>( + &'a self, + _services: &'a RuntimeServices, + job: &'a MediaJob<'a>, + ) -> MediaSigning<'a> { + Box::pin(async move { + self.jobs + .lock() + .expect("should lock jobs") + .push(RecordedJob { + reader_url: job.reader_url.to_owned(), + content_type: job.content_type.to_owned(), + source: job.source.to_vec(), + source_sha256: job.source_sha256.to_owned(), + source_reference: job.source_reference.map(str::to_owned), + source_origin_host: job.source_origin_host.map(str::to_owned), + optimizer_output: job.optimizer_output, + }); + // Signs under the namespace current when it signs, as a real + // signer reports the policy it used. + let namespace = self + .namespace + .lock() + .expect("should lock namespace") + .clone() + .unwrap_or_default(); + let mut bytes = job.source.to_vec(); + bytes.extend_from_slice(SIGNED_SUFFIX); + match self.behavior { + Behavior::Fail => Err(MediaSigningFailure { + reason: "stub_failure", + }), + Behavior::Sign => Ok(SignedMedia { + bytes: Bytes::from(bytes), + content_type: HeaderValue::from_static("image/png"), + headers: vec![( + HeaderName::from_static(MANIFEST_HEADER), + HeaderValue::from_static("instance-1"), + )], + namespace, + }), + Behavior::SignWithLink => Ok(SignedMedia { + bytes: Bytes::from(bytes), + content_type: HeaderValue::from_static("image/png"), + headers: vec![(header::LINK, HeaderValue::from_static(MANIFEST_LINK))], + namespace, + }), + Behavior::ReservedHeader => Ok(SignedMedia { + bytes: Bytes::from(bytes), + content_type: HeaderValue::from_static("image/png"), + headers: vec![(header::ETAG, HeaderValue::from_static("\"forged\""))], + namespace, + }), + } + }) + } +} diff --git a/crates/trusted-server-core/src/media_signing/tests.rs b/crates/trusted-server-core/src/media_signing/tests.rs new file mode 100644 index 0000000000..da92c92291 --- /dev/null +++ b/crates/trusted-server-core/src/media_signing/tests.rs @@ -0,0 +1,1544 @@ +use std::sync::Arc; + +use base64::Engine as _; +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use futures::StreamExt as _; +use http::{HeaderMap, HeaderName, HeaderValue, Method, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; + +use super::test_support::{Behavior, MANIFEST_LINK, SIGNED_SUFFIX, StubSigner}; +use super::*; +use crate::platform::test_support::{MemoryLoadingCache, noop_services}; +use crate::test_support::tests::create_test_settings; + +const SOURCE: &[u8] = b"origin-image-bytes"; +const PATH: &str = "/media/42"; +const FETCHED_URL: &str = "https://origin.example.com/media/42"; +const READER_URL: &str = "https://www.example.com/media/42"; +const ORIGIN_IDENTITY: &str = "https://origin.example.com\0origin.example.com"; +const ORIGIN_ETAG: &str = "\"origin-etag\""; + +fn identity() -> VariantIdentity<'static> { + VariantIdentity { + reader_url: READER_URL, + origin_identity: ORIGIN_IDENTITY, + fetched_url: FETCHED_URL, + reader_query_free: true, + } +} + +fn signed_bytes() -> Vec { + let mut bytes = SOURCE.to_vec(); + bytes.extend_from_slice(SIGNED_SUFFIX); + bytes +} + +fn signed_etag() -> String { + format!("\"sig-{}\"", hex::encode(Sha256::digest(signed_bytes()))) +} + +fn request_headers(pairs: &[(HeaderName, &str)]) -> HeaderMap { + let mut headers = HeaderMap::new(); + for (name, value) in pairs { + headers.append( + name.clone(), + HeaderValue::from_str(value).expect("should build header"), + ); + } + headers +} + +/// The headers of a browser asking for an image, which may be answered +/// from a kept copy. +fn image_request(pairs: &[(HeaderName, &str)]) -> HeaderMap { + let mut headers = request_headers(pairs); + headers.insert("sec-fetch-dest", HeaderValue::from_static("image")); + headers +} + +fn origin_response( + status: StatusCode, + body: EdgeBody, + extra: &[(HeaderName, &str)], +) -> Response { + let mut builder = Response::builder() + .status(status) + .header(header::CONTENT_TYPE, "image/png") + .header(header::ETAG, ORIGIN_ETAG) + .header(header::CACHE_CONTROL, "public, max-age=600"); + for (name, value) in extra { + builder = builder.header(name.clone(), *value); + } + builder.body(body).expect("should build origin response") +} + +fn whole_origin() -> Response { + origin_response(StatusCode::OK, EdgeBody::from(SOURCE.to_vec()), &[]) +} + +fn part_origin() -> Response { + origin_response( + StatusCode::PARTIAL_CONTENT, + EdgeBody::from(SOURCE[0..=3].to_vec()), + &[(header::CONTENT_RANGE, "bytes 0-3/18")], + ) +} + +async fn body_bytes(body: EdgeBody) -> Vec { + match body { + EdgeBody::Once(bytes) => bytes.to_vec(), + EdgeBody::Stream(mut stream) => { + let mut collected = Vec::new(); + while let Some(chunk) = stream.next().await { + collected.extend_from_slice(&chunk.expect("should read stream chunk")); + } + collected + } + } +} + +fn header<'a>(response: &'a Response, name: &str) -> Option<&'a str> { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) +} + +fn claim_shared( + services: &RuntimeServices, + method: &Method, + headers: &HeaderMap, + shared: bool, +) -> MediaClaim { + MediaClaim::of_request( + &create_test_settings(), + services, + method, + PATH, + headers, + shared, + ) + .expect("should claim a media path") +} + +fn claim(services: &RuntimeServices, method: &Method, headers: &HeaderMap) -> MediaClaim { + claim_shared(services, method, headers, true) +} + +fn keeping(signer: &Arc) -> RuntimeServices { + signer + .installed_in(noop_services()) + .with_loading_cache(Arc::new(MemoryLoadingCache::default())) +} + +fn split(response: Response) -> (Response, Vec) { + let (parts, body) = response.into_parts(); + let bytes = futures::executor::block_on(body_bytes(body)); + (Response::from_parts(parts, EdgeBody::empty()), bytes) +} + +/// What `deliver` decides for the origin's `response`, with the facts its +/// own headers give. +fn delivery( + claim: &MediaClaim, + services: &RuntimeServices, + response: Response, +) -> Delivery { + let facts = OriginFacts::of(response.headers()); + futures::executor::block_on(claim.deliver( + services, + &identity(), + Answered::Origin(facts), + response, + )) +} + +/// Delivers the origin's `response` to a reader whose request is `method` +/// with `headers`, expecting core to send it without a second fetch. +fn deliver( + signer: &Arc, + method: &Method, + headers: &HeaderMap, + response: Response, +) -> (Response, Vec) { + let services = signer.installed_in(noop_services()); + let claim = claim(&services, method, headers); + let Delivery::Send(response) = delivery(&claim, &services, response) else { + panic!("should send without asking the origin again"); + }; + split(response) +} + +#[test] +fn signs_the_exact_captured_bytes_and_owns_the_validators() { + let signer = StubSigner::new(Behavior::Sign, 1024); + + let (response, body) = deliver(&signer, &Method::GET, &HeaderMap::new(), whole_origin()); + + assert_eq!(response.status(), StatusCode::OK, "should answer 200"); + assert_eq!( + body, + signed_bytes(), + "should send the signer's bytes unchanged" + ); + let jobs = signer.jobs(); + assert_eq!(jobs.len(), 1, "should ask the signer once"); + assert_eq!( + jobs[0].source, SOURCE, + "should hand the signer the exact origin bytes" + ); + assert_eq!(jobs[0].source_sha256, hex::encode(Sha256::digest(SOURCE))); + assert_eq!( + jobs[0].reader_url, READER_URL, + "should name the reader's address" + ); + assert_eq!( + jobs[0].content_type, "image/png", + "should pass the origin type" + ); + assert_eq!(jobs[0].source_reference.as_deref(), Some(FETCHED_URL)); + assert_eq!( + jobs[0].source_origin_host.as_deref(), + Some("origin.example.com") + ); + assert_eq!(header(&response, "etag"), Some(signed_etag().as_str())); + assert_eq!( + header(&response, "repr-digest"), + Some( + format!( + "sha-256=:{}:", + base64::engine::general_purpose::STANDARD.encode(Sha256::digest(signed_bytes())) + ) + .as_str() + ), + "should describe the signed representation's digest" + ); + assert_eq!( + header(&response, "content-length"), + Some(signed_bytes().len().to_string().as_str()), + "should frame the signed length" + ); + assert_eq!( + header(&response, "accept-ranges"), + Some("none"), + "a signed representation nothing keeps offers no ranges" + ); + assert_eq!(header(&response, "x-test-manifest"), Some("instance-1")); + assert_eq!( + header(&response, "cache-control"), + Some("public, max-age=600") + ); +} + +#[test] +fn merges_the_signers_link_with_the_origins_links() { + let signer = StubSigner::new(Behavior::SignWithLink, 1024); + let origin_link = "; rel=preload; as=style"; + + let (response, _) = deliver( + &signer, + &Method::GET, + &HeaderMap::new(), + origin_response( + StatusCode::OK, + EdgeBody::from(SOURCE.to_vec()), + &[(header::LINK, origin_link)], + ), + ); + + let links: Vec<&str> = response + .headers() + .get_all(header::LINK) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect(); + assert_eq!( + links, + vec![origin_link, MANIFEST_LINK], + "should keep the origin's link and add the manifest's" + ); +} + +#[test] +fn names_the_origin_host_even_where_the_reference_is_withheld() { + for (fetched_url, host) in [ + ( + "https://origin.example.com/media/42?signature=secret", + "origin.example.com", + ), + ( + "https://user:pass@Origin.Example.com/media/42", + "origin.example.com", + ), + ( + "https://origin.example.com:8443/media/42#part", + "origin.example.com", + ), + ] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let claim = claim(&services, &Method::GET, &HeaderMap::new()); + let response = whole_origin(); + let facts = OriginFacts::of(response.headers()); + futures::executor::block_on(claim.deliver( + &services, + &VariantIdentity { + fetched_url, + ..identity() + }, + Answered::Origin(facts), + response, + )); + let job = &signer.jobs()[0]; + assert_eq!( + job.source_reference, None, + "should send no reference for {fetched_url}" + ); + assert_eq!( + job.source_origin_host.as_deref(), + Some(host), + "should name the host of {fetched_url}" + ); + } +} + +#[test] +fn answers_a_single_range_from_the_signed_bytes() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let total = signed_bytes().len(); + + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, "bytes=2-5")]), + whole_origin(), + ); + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + body, + signed_bytes()[2..=5], + "should send the asked bytes of the signed body" + ); + assert_eq!( + header(&response, "content-range"), + Some(format!("bytes 2-5/{total}").as_str()) + ); + assert_eq!(header(&response, "content-length"), Some("4")); +} + +#[test] +fn answers_suffix_and_open_ranges_from_the_signed_bytes() { + let total = signed_bytes().len(); + for (range, first) in [("bytes=-4", total - 4), ("bytes=10-", 10)] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, range)]), + whole_origin(), + ); + assert_eq!( + response.status(), + StatusCode::PARTIAL_CONTENT, + "{range} should be partial" + ); + assert_eq!( + body, + signed_bytes()[first..], + "{range} should send the tail" + ); + assert_eq!( + header(&response, "content-range"), + Some(format!("bytes {first}-{}/{total}", total - 1).as_str()) + ); + } +} + +#[test] +fn refuses_a_range_past_the_signed_length() { + let signer = StubSigner::new(Behavior::Sign, 1024); + + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, "bytes=1000-")]), + whole_origin(), + ); + + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert!(body.is_empty(), "should send no body with a 416"); + assert_eq!( + header(&response, "content-range"), + Some(format!("bytes */{}", signed_bytes().len()).as_str()) + ); +} + +#[test] +fn answers_several_ranges_and_malformed_ranges_with_the_whole_signed_body() { + for range in ["bytes=0-1,4-5", "bytes=5-2", "items=0-1", "bytes=+1-2"] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, range)]), + whole_origin(), + ); + assert_eq!( + response.status(), + StatusCode::OK, + "{range} should be answered whole" + ); + assert_eq!(body, signed_bytes()); + } +} + +#[test] +fn honors_if_range_only_for_the_signed_entity_tag() { + let signed = signed_etag(); + for (if_range, expected) in [ + (signed.as_str(), StatusCode::PARTIAL_CONTENT), + (ORIGIN_ETAG, StatusCode::OK), + ("Tue, 01 Sep 2026 00:00:00 GMT", StatusCode::OK), + ] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, _) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, "bytes=0-3"), (header::IF_RANGE, if_range)]), + whole_origin(), + ); + assert_eq!(response.status(), expected, "If-Range {if_range}"); + } +} + +#[test] +fn tells_a_precondition_naming_signed_bytes_from_one_naming_the_origins() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let signed = signed_etag(); + for (headers, names_signed) in [ + ( + request_headers(&[ + (header::RANGE, "bytes=0-3"), + (header::IF_RANGE, signed.as_str()), + ]), + true, + ), + ( + request_headers(&[(header::IF_MATCH, format!("\"other\", {signed}").as_str())]), + true, + ), + ( + request_headers(&[ + (header::RANGE, "bytes=0-3"), + (header::IF_RANGE, ORIGIN_ETAG), + ]), + false, + ), + ( + request_headers(&[(header::IF_NONE_MATCH, signed.as_str())]), + false, + ), + ( + request_headers(&[(header::IF_MATCH, "W/\"sig-00\"")]), + false, + ), + ] { + assert_eq!( + claim(&services, &Method::GET, &headers).names_signed_bytes(), + names_signed, + "{headers:?}" + ); + } +} + +#[test] +fn answers_if_none_match_against_the_signed_tag_only() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[( + header::IF_NONE_MATCH, + format!("\"other\", W/{}", signed_etag()).as_str(), + )]), + whole_origin(), + ); + assert_eq!( + response.status(), + StatusCode::NOT_MODIFIED, + "a weak match of the signed tag is a 304" + ); + assert!(body.is_empty(), "a 304 carries no body"); + assert_eq!(header(&response, "content-length"), None); + assert_eq!(header(&response, "etag"), Some(signed_etag().as_str())); + assert_eq!( + header(&response, "x-test-manifest"), + Some("instance-1"), + "a 304 carries the signer's headers" + ); + + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::IF_NONE_MATCH, ORIGIN_ETAG)]), + whole_origin(), + ); + assert_eq!( + response.status(), + StatusCode::OK, + "the origin's tag never matches signed bytes" + ); + assert_eq!(body, signed_bytes()); +} + +#[test] +fn fails_if_match_that_names_another_representation() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::IF_MATCH, ORIGIN_ETAG)]), + whole_origin(), + ); + assert_eq!(response.status(), StatusCode::PRECONDITION_FAILED); + assert!(body.is_empty()); + + let signer = StubSigner::new(Behavior::Sign, 1024); + let (response, _) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::IF_MATCH, signed_etag().as_str())]), + whole_origin(), + ); + assert_eq!( + response.status(), + StatusCode::OK, + "the signed tag satisfies If-Match" + ); +} + +#[test] +fn signs_the_whole_behind_a_part_that_can_be_kept_and_ranges_the_signed_bytes() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + let headers = image_request(&[(header::RANGE, "bytes=0-3")]); + let claim = claim(&services, &Method::GET, &headers); + + let Delivery::FetchWhole(fallback) = delivery(&claim, &services, part_origin()) else { + panic!("should ask for the whole representation of a part it can keep"); + }; + assert!(signer.jobs().is_empty(), "a part is never signed"); + + let delivered = futures::executor::block_on(claim.deliver_whole( + &services, + &identity(), + whole_origin(), + fallback, + )); + let (response, body) = split(delivered); + assert_eq!( + signer.jobs()[0].source, + SOURCE, + "should sign the whole answer's exact bytes" + ); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + body, + signed_bytes()[0..=3], + "should range the signed bytes, not the origin's" + ); + assert_eq!(header(&response, "etag"), Some(signed_etag().as_str())); + assert_eq!( + header(&response, "accept-ranges"), + Some("bytes"), + "a kept representation offers ranges" + ); +} + +#[test] +fn passes_a_part_through_where_nothing_would_keep_the_signed_whole() { + let cases: [(RuntimeServices, Response, &str); 3] = [ + ( + StubSigner::new(Behavior::Sign, 1024).installed_in(noop_services()), + part_origin(), + "no cache keeps values", + ), + ( + keeping(&StubSigner::new(Behavior::Sign, 1024)), + origin_response( + StatusCode::PARTIAL_CONTENT, + EdgeBody::from(SOURCE[0..=3].to_vec()), + &[ + (header::CONTENT_RANGE, "bytes 0-3/18"), + (header::CACHE_CONTROL, "no-cache"), + ], + ), + "the origin gave no shared freshness", + ), + ( + keeping(&StubSigner::new(Behavior::Sign, 1024)), + origin_response( + StatusCode::PARTIAL_CONTENT, + EdgeBody::from(b"part".to_vec()), + &[(header::CONTENT_RANGE, "bytes 0-3/50000000")], + ), + "the whole is over the limit", + ), + ]; + for (services, part, case) in cases { + let claim = claim( + &services, + &Method::GET, + &image_request(&[(header::RANGE, "bytes=0-3")]), + ); + let Delivery::Send(response) = delivery(&claim, &services, part) else { + panic!("{case}: should not read the whole origin for a part"); + }; + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT, "{case}"); + assert_eq!( + header(&response, "etag"), + Some(ORIGIN_ETAG), + "{case}: keeps the origin's part" + ); + } +} + +#[test] +fn asks_for_the_signed_tail_the_origin_refused_as_past_its_own_length() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let tail = signed_bytes().len() - 2; + let range = format!("bytes={tail}-"); + let headers = request_headers(&[(header::RANGE, range.as_str())]); + let claim = claim(&services, &Method::GET, &headers); + let refusal = origin_response( + StatusCode::RANGE_NOT_SATISFIABLE, + EdgeBody::empty(), + &[(header::CONTENT_RANGE, "bytes */18")], + ); + + let Delivery::FetchWhole(fallback) = delivery(&claim, &services, refusal) else { + panic!("a range past the origin's length may name the signed tail"); + }; + let delivered = futures::executor::block_on(claim.deliver_whole( + &services, + &identity(), + whole_origin(), + fallback, + )); + let (response, body) = split(delivered); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(body, signed_bytes()[tail..], "should send the signed tail"); +} + +#[test] +fn sends_the_origins_part_without_validators_when_the_whole_fetch_cannot_be_signed() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + let claim = claim( + &services, + &Method::GET, + &image_request(&[(header::RANGE, "bytes=0-3")]), + ); + + let mut missing = whole_origin(); + *missing.status_mut() = StatusCode::NOT_FOUND; + let delivered = futures::executor::block_on(claim.deliver_whole( + &services, + &identity(), + missing, + part_origin(), + )); + let (response, body) = split(delivered); + assert_eq!( + response.status(), + StatusCode::PARTIAL_CONTENT, + "should keep the origin's part" + ); + assert_eq!(body, SOURCE[0..=3]); + assert_eq!( + header(&response, "etag"), + None, + "an unsigned fallback carries no validator to pin it" + ); + assert_eq!( + header(&response, "cache-control"), + Some("private, max-age=60") + ); + assert!(signer.jobs().is_empty()); +} + +#[test] +fn answers_head_from_the_signed_whole_with_no_body() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + let claim = claim(&services, &Method::HEAD, &image_request(&[])); + let head = origin_response( + StatusCode::OK, + EdgeBody::empty(), + &[(header::CONTENT_LENGTH, "18")], + ); + + let Delivery::FetchWhole(fallback) = delivery(&claim, &services, head) else { + panic!("a HEAD of a bounded type that can be kept should ask for the whole body"); + }; + let delivered = futures::executor::block_on(claim.deliver_whole( + &services, + &identity(), + whole_origin(), + fallback, + )); + let (response, body) = split(delivered); + assert_eq!(response.status(), StatusCode::OK); + assert!(body.is_empty(), "a HEAD carries no body"); + assert_eq!( + header(&response, "content-length"), + Some(signed_bytes().len().to_string().as_str()), + "a HEAD advertises the signed length" + ); + assert_eq!(header(&response, "etag"), Some(signed_etag().as_str())); +} + +#[test] +fn never_signs_a_response_the_origin_marked_as_one_readers() { + let private_markers: [(HeaderName, &str); 4] = [ + (header::SET_COOKIE, "session=1"), + (header::CACHE_CONTROL, "private, max-age=60"), + (header::VARY, "Accept, Cookie"), + (header::VARY, "*"), + ]; + for (name, value) in private_markers { + let signer = StubSigner::new(Behavior::Sign, 1024); + let mut response = whole_origin(); + response + .headers_mut() + .insert(name.clone(), HeaderValue::from_static(value)); + let (response, body) = deliver(&signer, &Method::GET, &HeaderMap::new(), response); + assert!( + signer.jobs().is_empty(), + "{name}: {value} should not reach the signer" + ); + assert_eq!( + body, SOURCE, + "{name}: {value} should go out as the origin sent it" + ); + assert_eq!(header(&response, "etag"), Some(ORIGIN_ETAG)); + } +} + +#[test] +fn reads_the_origins_own_headers_not_the_policy_core_wrote_over_them() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let claim = claim(&services, &Method::GET, &HeaderMap::new()); + let raw = origin_response( + StatusCode::OK, + EdgeBody::empty(), + &[(header::CACHE_CONTROL, "public, max-age=60, no-transform")], + ); + let facts = OriginFacts::of(raw.headers()); + let normalized = origin_response( + StatusCode::OK, + EdgeBody::from(SOURCE.to_vec()), + &[(header::CACHE_CONTROL, "public, max-age=31536000, immutable")], + ); + + futures::executor::block_on(claim.deliver( + &services, + &identity(), + Answered::Origin(facts), + normalized, + )); + assert!( + signer.jobs().is_empty(), + "the origin's no-transform stands under core's policy" + ); +} + +#[test] +fn claims_nothing_without_a_signer_with_authorization_or_off_its_paths() { + let settings = create_test_settings(); + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let none = |services: &RuntimeServices, method: &Method, path: &str, headers: &HeaderMap| { + MediaClaim::of_request(&settings, services, method, path, headers, true).is_none() + }; + assert!( + none(&noop_services(), &Method::GET, PATH, &HeaderMap::new()), + "services without a signer claim nothing" + ); + assert!( + none( + &services, + &Method::GET, + PATH, + &request_headers(&[(header::AUTHORIZATION, "Bearer reader-token")]) + ), + "a request bound to one reader's credentials is never claimed" + ); + assert!( + none(&services, &Method::GET, "/api/items", &HeaderMap::new()), + "a path the signer leaves alone" + ); + assert!( + none(&services, &Method::POST, PATH, &HeaderMap::new()), + "only GET and HEAD are claimed" + ); +} + +#[test] +fn tells_a_media_request_by_destination_accept_or_extension() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let settings = create_test_settings(); + let looks = |path: &str, headers: &HeaderMap| { + MediaClaim::of_request(&settings, &services, &Method::GET, path, headers, true) + .expect("should claim") + .looks_like_media() + }; + assert!( + looks(PATH, &image_request(&[])), + "an image destination is media" + ); + assert!(!looks( + "/photo.jpg", + &request_headers(&[("sec-fetch-dest".parse().expect("should parse"), "script")]) + )); + assert!(looks( + PATH, + &request_headers(&[(header::ACCEPT, "image/avif,image/webp,*/*")]) + )); + assert!( + looks("/clips/intro.MP4", &HeaderMap::new()), + "a media extension is media" + ); + assert!( + !looks(PATH, &request_headers(&[(header::ACCEPT, "text/html,*/*")])), + "a page is not" + ); +} + +#[test] +fn leaves_untransformable_encoded_and_unhandled_bodies_as_the_origin_sent_them() { + let extras: [(HeaderName, &str); 3] = [ + (header::CACHE_CONTROL, "public, no-transform"), + (header::CONTENT_ENCODING, "gzip"), + (header::CONTENT_TYPE, "text/plain"), + ]; + for (name, value) in extras { + let signer = StubSigner::new(Behavior::Sign, 1024); + let mut response = whole_origin(); + response + .headers_mut() + .insert(name.clone(), HeaderValue::from_static(value)); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, "bytes=0-1")]), + response, + ); + assert!( + signer.jobs().is_empty(), + "{name}: {value} should not reach the signer" + ); + assert_eq!( + response.status(), + StatusCode::OK, + "{name}: {value} keeps the origin's status" + ); + assert_eq!(body, SOURCE); + } +} + +#[test] +fn passes_other_statuses_through_untouched() { + for status in [ + StatusCode::NOT_FOUND, + StatusCode::NOT_MODIFIED, + StatusCode::RANGE_NOT_SATISFIABLE, + ] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let response = origin_response(status, EdgeBody::empty(), &[]); + let (response, _) = deliver(&signer, &Method::GET, &HeaderMap::new(), response); + assert_eq!( + response.status(), + status, + "a reader that asked no range gets the origin's {status}" + ); + assert_eq!(header(&response, "etag"), Some(ORIGIN_ETAG)); + assert!(signer.jobs().is_empty()); + } +} + +#[test] +fn sends_the_origin_bytes_unsigned_without_validators_when_signing_fails() { + for behavior in [Behavior::Fail, Behavior::ReservedHeader] { + let signer = StubSigner::new(behavior, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &request_headers(&[(header::RANGE, "bytes=0-5")]), + whole_origin(), + ); + assert_eq!( + signer.jobs().len(), + 1, + "{behavior:?} should have asked the signer" + ); + assert_eq!( + response.status(), + StatusCode::OK, + "{behavior:?} keeps the origin's own answer" + ); + assert_eq!(body, SOURCE); + assert_eq!( + header(&response, "etag"), + None, + "{behavior:?} drops the origin's validator" + ); + assert_eq!( + header(&response, "cache-control"), + Some("private, max-age=60") + ); + assert_eq!(header(&response, "x-test-manifest"), None); + assert_eq!(header(&response, "repr-digest"), None); + } +} + +#[test] +fn captures_a_streamed_body_within_the_limit() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let chunks = SOURCE + .chunks(5) + .map(Bytes::copy_from_slice) + .collect::>(); + let (response, body) = deliver( + &signer, + &Method::GET, + &HeaderMap::new(), + origin_response( + StatusCode::OK, + EdgeBody::stream(futures::stream::iter(chunks)), + &[], + ), + ); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + signer.jobs()[0].source, + SOURCE, + "should join the streamed chunks exactly" + ); + assert_eq!(body, signed_bytes()); +} + +#[test] +fn streams_a_body_over_the_limit_unsigned_and_whole() { + let signer = StubSigner::new(Behavior::Sign, 8); + let chunks = SOURCE + .chunks(4) + .map(Bytes::copy_from_slice) + .collect::>(); + let (response, body) = deliver( + &signer, + &Method::GET, + &HeaderMap::new(), + origin_response( + StatusCode::OK, + EdgeBody::stream(futures::stream::iter(chunks)), + &[], + ), + ); + assert!( + signer.jobs().is_empty(), + "an oversized body should not reach the signer" + ); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + body, SOURCE, + "should send every byte read and unread, in order" + ); + assert_eq!(header(&response, "etag"), Some(ORIGIN_ETAG)); +} + +#[test] +fn keeps_a_failed_stream_failing_after_the_bytes_read() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = signer.installed_in(noop_services()); + let claim = claim(&services, &Method::GET, &HeaderMap::new()); + let stream = futures::stream::iter(vec![ + Ok(Bytes::from_static(b"first")), + Err(std::io::Error::other("origin reset")), + ]); + let Delivery::Send(delivered) = delivery( + &claim, + &services, + origin_response(StatusCode::OK, EdgeBody::from_stream(stream), &[]), + ) else { + panic!("should send the origin's answer"); + }; + let EdgeBody::Stream(mut body) = delivered.into_body() else { + panic!("should keep a failed body streaming"); + }; + futures::executor::block_on(async { + let first = body.next().await.expect("should yield the bytes read"); + assert_eq!( + first.expect("should read the first chunk"), + Bytes::from_static(b"first") + ); + assert!( + body.next() + .await + .expect("should yield the failure") + .is_err(), + "should hand the reader the origin's failure, never a short body as whole" + ); + }); + assert!( + signer.jobs().is_empty(), + "a body that failed is never signed" + ); +} + +/// Signs once through `services` for a reader whose request is `headers`, +/// then gives back what a later image request finds before the origin. +fn sign_then_look_up( + signer: &Arc, + services: &RuntimeServices, + headers: &HeaderMap, + shared: bool, + origin: Response, +) -> Option> { + let first = claim_shared(services, &Method::GET, headers, shared); + let _ = delivery(&first, services, origin); + assert_eq!(signer.jobs().len(), 1, "should sign the first request"); + let later = claim(services, &Method::GET, &image_request(&[])); + futures::executor::block_on(later.kept(services, &identity())) +} + +#[test] +fn answers_a_later_range_from_the_kept_signed_representation_before_the_origin() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + + let kept = sign_then_look_up( + &signer, + &services, + &image_request(&[]), + true, + whole_origin(), + ) + .expect("should keep a fresh signed representation"); + let age = header(&kept, "age") + .and_then(|age| age.parse::().ok()) + .expect("a kept copy carries its age"); + assert!( + age <= 1, + "a copy kept just now has almost no age, got {age}" + ); + + let ranged = image_request(&[(header::RANGE, "bytes=1-4")]); + let claim = claim(&services, &Method::GET, &ranged); + let Delivery::Send(response) = + futures::executor::block_on(claim.deliver(&services, &identity(), Answered::Kept, kept)) + else { + panic!("a kept representation is answered at once"); + }; + let (response, body) = split(response); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(body, signed_bytes()[1..=4]); + assert_eq!(header(&response, "etag"), Some(signed_etag().as_str())); + assert_eq!(header(&response, "x-test-manifest"), Some("instance-1")); + assert_eq!(signer.jobs().len(), 1, "the kept copy is not signed again"); +} + +#[test] +fn keeps_and_serves_nothing_for_a_request_whose_answer_could_be_one_readers() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + assert!( + sign_then_look_up( + &signer, + &services, + &image_request(&[]), + false, + whole_origin() + ) + .is_none(), + "an answer to a cookie-bearing request is never kept for other readers" + ); + + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + assert!( + sign_then_look_up( + &signer, + &services, + &image_request(&[]), + true, + whole_origin() + ) + .is_some() + ); + let cookie_bearing = claim_shared(&services, &Method::GET, &image_request(&[]), false); + assert!( + futures::executor::block_on(cookie_bearing.kept(&services, &identity())).is_none(), + "a cookie-bearing request is never answered from a copy kept for others" + ); + let revalidating = claim( + &services, + &Method::GET, + &image_request(&[(header::CACHE_CONTROL, "no-cache")]), + ); + assert!( + futures::executor::block_on(revalidating.kept(&services, &identity())).is_none(), + "a request asking for revalidation is not answered from a kept copy" + ); + let not_media = claim(&services, &Method::GET, &HeaderMap::new()); + assert!( + futures::executor::block_on(not_media.kept(&services, &identity())).is_none(), + "a request that does not look like media is not looked up" + ); +} + +#[test] +fn serves_nothing_kept_once_the_signers_namespace_changes_or_fails() { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + assert!( + sign_then_look_up( + &signer, + &services, + &image_request(&[]), + true, + whole_origin() + ) + .is_some() + ); + let claim = claim(&services, &Method::GET, &image_request(&[])); + + signer.set_namespace(Ok("policy-2".to_owned())); + assert!( + futures::executor::block_on(claim.kept(&services, &identity())).is_none(), + "a changed policy finds nothing kept under the old one" + ); + + signer.set_namespace(Err(MediaSigningFailure { reason: "revoked" })); + assert!( + futures::executor::block_on(claim.kept(&services, &identity())).is_none(), + "a revoked signer serves nothing kept" + ); +} + +#[test] +fn keeps_nothing_the_origin_did_not_make_shareably_fresh() { + for cache_control in ["no-cache", "public", "public, max-age=0"] { + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + let mut origin = whole_origin(); + origin.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static(cache_control), + ); + assert!( + sign_then_look_up(&signer, &services, &image_request(&[]), true, origin).is_none(), + "`{cache_control}` should keep no signed copy" + ); + } + + let signer = StubSigner::new(Behavior::Sign, 1024); + let services = keeping(&signer); + let mut origin = whole_origin(); + origin + .headers_mut() + .insert(header::VARY, HeaderValue::from_static("Accept")); + assert!( + sign_then_look_up(&signer, &services, &image_request(&[]), true, origin).is_none(), + "an answer that varies on the reader's Accept keeps no single signed copy" + ); +} + +#[test] +fn a_kept_copy_ages_and_is_refused_when_its_body_is_not_the_one_signed() { + let mut headers = HeaderMap::new(); + let digest = Sha256::digest(signed_bytes()); + headers.insert( + header::ETAG, + HeaderValue::from_str(&signed_entity_tag(digest.as_slice())).expect("should build tag"), + ); + let body = Bytes::from(signed_bytes()); + let encoded = encode_kept(&KeptRecord { + kept_at: 1_000, + age: 5, + headers: &headers, + body: &body, + }); + + let kept = decode_kept(&encoded, 1_030).expect("should read an intact copy"); + assert_eq!( + header(&kept, "age"), + Some("35"), + "should add the time kept to the origin's age" + ); + + let mut altered = encoded.to_vec(); + let last = altered.len() - 1; + altered[last] ^= 0x01; + assert!( + decode_kept(&Bytes::from(altered), 1_030).is_none(), + "a body that no longer hashes to its tag is never served" + ); +} + +#[test] +fn strips_every_header_that_shortens_the_origin_answer() { + let mut headers = request_headers(&[ + (header::RANGE, "bytes=0-1"), + (header::IF_RANGE, ORIGIN_ETAG), + (header::IF_NONE_MATCH, ORIGIN_ETAG), + (header::IF_MODIFIED_SINCE, "Tue, 01 Sep 2026 00:00:00 GMT"), + (header::IF_MATCH, ORIGIN_ETAG), + (header::IF_UNMODIFIED_SINCE, "Tue, 01 Sep 2026 00:00:00 GMT"), + (header::ACCEPT, "image/avif"), + ]); + strip_representation_preconditions(&mut headers); + assert_eq!(headers.len(), 1, "should keep only the unrelated header"); + assert!(headers.contains_key(header::ACCEPT)); +} + +/// A manifest the origin already links its bytes to. +const ORIGIN_MANIFEST_LINK: &str = "; rel=\"c2pa-manifest\""; + +/// `response` with the origin's own manifest `Link`. +fn linked(mut response: Response) -> Response { + response + .headers_mut() + .append(header::LINK, HeaderValue::from_static(ORIGIN_MANIFEST_LINK)); + response +} + +fn links(response: &Response) -> Vec { + response + .headers() + .get_all(header::LINK) + .iter() + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + .collect() +} + +#[test] +fn keeps_an_origin_manifest_link_and_its_bytes_for_external_and_embeddable_signers() { + // `SignWithLink` stands for an external-only signer, `Sign` for one + // that embeds a manifest and changes the bytes. + for behavior in [Behavior::SignWithLink, Behavior::Sign] { + let signer = StubSigner::new(behavior, 1024); + let (response, body) = deliver( + &signer, + &Method::GET, + &image_request(&[]), + linked(whole_origin()), + ); + assert!( + signer.jobs().is_empty(), + "{behavior:?}: should never sign bytes that name a manifest of their own" + ); + assert_eq!(body, SOURCE, "{behavior:?}: should send the origin's bytes"); + assert_eq!( + header(&response, "etag"), + Some(ORIGIN_ETAG), + "{behavior:?}: should keep the origin's validator" + ); + assert_eq!( + links(&response), + vec![ORIGIN_MANIFEST_LINK.to_owned()], + "{behavior:?}: should keep only the origin's manifest link" + ); + assert_eq!(header(&response, "x-test-manifest"), None); + assert_eq!(header(&response, "repr-digest"), None); + } +} + +#[test] +fn never_signs_a_whole_fetch_where_an_answer_names_an_origin_manifest() { + let signer = StubSigner::new(Behavior::SignWithLink, 1024); + let services = keeping(&signer); + let first_claim = claim( + &services, + &Method::GET, + &image_request(&[(header::RANGE, "bytes=0-3")]), + ); + let Delivery::Send(response) = delivery(&first_claim, &services, linked(part_origin())) else { + panic!("should never ask for the whole behind a part that names an origin manifest"); + }; + let (response, body) = split(response); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(body, SOURCE[0..=3]); + assert_eq!(links(&response), vec![ORIGIN_MANIFEST_LINK.to_owned()]); + assert!(signer.jobs().is_empty()); + + let tail = signed_bytes().len() - 2; + let range = format!("bytes={tail}-"); + for (refusal_linked, whole_linked) in [(false, true), (true, false)] { + let signer = StubSigner::new(Behavior::SignWithLink, 1024); + let services = signer.installed_in(noop_services()); + let claim = claim( + &services, + &Method::GET, + &request_headers(&[(header::RANGE, range.as_str())]), + ); + let refusal = origin_response( + StatusCode::RANGE_NOT_SATISFIABLE, + EdgeBody::empty(), + &[(header::CONTENT_RANGE, "bytes */18")], + ); + let refusal = if refusal_linked { + linked(refusal) + } else { + refusal + }; + let Delivery::FetchWhole(fallback) = delivery(&claim, &services, refusal) else { + panic!("a range past the origin's length may name the signed tail"); + }; + let whole = if whole_linked { + linked(whole_origin()) + } else { + whole_origin() + }; + let delivered = futures::executor::block_on(claim.deliver_whole( + &services, + &identity(), + whole, + fallback, + )); + let (response, body) = split(delivered); + let case = format!("refusal_linked={refusal_linked} whole_linked={whole_linked}"); + assert!(signer.jobs().is_empty(), "{case}: should sign nothing"); + assert_eq!( + response.status(), + StatusCode::RANGE_NOT_SATISFIABLE, + "{case}: should send the origin's own answer" + ); + assert!(body.is_empty(), "{case}"); + assert_eq!(header(&response, "etag"), Some(ORIGIN_ETAG), "{case}"); + let expected = if refusal_linked { + vec![ORIGIN_MANIFEST_LINK.to_owned()] + } else { + Vec::new() + }; + assert_eq!( + links(&response), + expected, + "{case}: should add no manifest link" + ); + } +} + +/// One ISO BMFF box of `kind` around `body`, with a 32-bit size. +fn bmff_box(kind: &[u8; 4], body: &[u8]) -> Vec { + let size = u32::try_from(body.len() + 8).expect("should fit a 32-bit size"); + let mut bytes = size.to_be_bytes().to_vec(); + bytes.extend_from_slice(kind); + bytes.extend_from_slice(body); + bytes +} + +fn concat(parts: &[Vec]) -> Vec { + parts.concat() +} + +/// A C2PA verifiable segment info `emsg`, as a packager writes it. +fn c2pa_emsg() -> Vec { + bmff_box(b"emsg", b"\0\0\0\0urn:c2pa:verifiable-segment-info\0fseg\0") +} + +#[test] +fn tells_the_parts_of_a_fragmented_stream_from_a_progressive_file() { + let init = concat(&[ + bmff_box(b"ftyp", b"cmfc\0\0\0\0iso6cmfc"), + bmff_box( + b"moov", + &concat(&[ + bmff_box(b"mvhd", &[0; 100]), + bmff_box(b"trak", &bmff_box(b"tkhd", &[0; 84])), + bmff_box(b"mvex", &bmff_box(b"trex", &[0; 24])), + bmff_box(b"pssh", &[0; 32]), + ]), + ), + ]); + let fragment = || { + vec![ + bmff_box(b"sidx", &[0; 32]), + bmff_box(b"moof", &bmff_box(b"mfhd", &[0; 8])), + bmff_box(b"mdat", b"coded-samples"), + ] + }; + let segment = concat(&[vec![bmff_box(b"styp", b"msdh\0\0\0\0msdhmsix")], fragment()].concat()); + let segment_with_emsg = concat( + &[ + vec![bmff_box(b"styp", b"msdh\0\0\0\0msdhmsix"), c2pa_emsg()], + fragment(), + ] + .concat(), + ); + let chunk_without_styp = concat(&fragment()); + let progressive = concat(&[ + bmff_box(b"ftyp", b"isom\0\0\0\0isomavc1"), + bmff_box(b"moov", &bmff_box(b"trak", &[0; 16])), + bmff_box(b"mdat", b"coded-samples"), + ]); + let fragmented_file = concat(&[ + bmff_box(b"ftyp", b"iso6\0\0\0\0iso6dash"), + bmff_box(b"moov", &bmff_box(b"mvex", &[0; 8])), + bmff_box(b"moof", &[0; 8]), + bmff_box(b"mdat", b"coded-samples"), + ]); + let mut large_size = init.clone(); + // The same init with its `ftyp` given a 64-bit size. + let ftyp_body = b"cmfc\0\0\0\0iso6cmfc"; + let mut large_ftyp = 1_u32.to_be_bytes().to_vec(); + large_ftyp.extend_from_slice(b"ftyp"); + large_ftyp.extend_from_slice(&(16 + ftyp_body.len() as u64).to_be_bytes()); + large_ftyp.extend_from_slice(ftyp_body); + large_size.splice(0..8 + ftyp_body.len(), large_ftyp); + let mut truncated = segment.clone(); + truncated.truncate(segment.len() - 3); + let mut zero_sized_last = segment.clone(); + let mdat_at = zero_sized_last.len() - (8 + b"coded-samples".len()); + zero_sized_last[mdat_at..mdat_at + 4].copy_from_slice(&0_u32.to_be_bytes()); + let too_many = bmff_box(b"free", &[]).repeat(MAX_BMFF_BOXES + 1); + + for (case, bytes, expected) in [ + ("Shaka-style init", &init, true), + ("init with a 64-bit ftyp size", &large_size, true), + ("media segment", &segment, true), + ("media segment with a C2PA emsg", &segment_with_emsg, true), + ("CMAF chunk without styp", &chunk_without_styp, true), + ("segment whose mdat runs to the end", &zero_sized_last, true), + ("progressive file", &progressive, false), + ("fragmented file with its media", &fragmented_file, false), + ("truncated segment", &truncated, false), + ("too many boxes", &too_many, false), + ("an image", &SOURCE.to_vec(), false), + ("nothing", &Vec::new(), false), + ] { + assert_eq!( + is_fragmented_stream_part(bytes), + expected, + "{case}: should be told correctly" + ); + } +} + +#[test] +fn origin_freshness_keeps_browser_and_shared_age_and_processing_independent() { + let receipt_seconds = 1_700_000_000; + let receipt = UNIX_EPOCH + Duration::from_secs(receipt_seconds); + for (control, age, date_age, elapsed, browser, shared) in [ + ( + "public, max-age=60, s-maxage=3600", + Some("0"), + None, + 0, + 60, + 3600, + ), + ( + "public, max-age=60, s-maxage=3600", + Some("5"), + Some(120), + 0, + 0, + 3480, + ), + ( + "public, max-age=60, s-maxage=3600", + None, + Some(120), + 0, + 0, + 3480, + ), + ( + "public, max-age=60, s-maxage=3600", + Some("200"), + Some(120), + 0, + 0, + 3400, + ), + ( + "public, max-age=90, s-maxage=180", + Some("10"), + None, + 40, + 40, + 130, + ), + ( + "public, max-age=90, s-maxage=180", + Some("10"), + None, + 90, + 0, + 80, + ), + ( + "public, max-age=90, s-maxage=10", + Some("15"), + None, + 0, + 75, + 0, + ), + ("public, max-age=90", Some("70"), None, 0, 20, 20), + ("public, max-age=90", Some("100"), None, 0, 0, 0), + ("public, s-maxage=90", None, None, 0, 0, 90), + ("public, max-age=bad, s-maxage=90", None, None, 0, 0, 90), + ("public, max-age=60, s-maxage=bad", None, None, 0, 60, 0), + ("public, max-age=60, s-maxage", None, None, 0, 60, 0), + ("public, max-age=60, max-age=90", None, None, 0, 0, 0), + ("public, max-age=60", Some("bad"), None, 0, 0, 0), + ("no-cache, max-age=90", Some("0"), None, 0, 0, 0), + ("public", Some("0"), None, 0, 0, 0), + ] { + let mut headers = HeaderMap::new(); + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_str(control).expect("control"), + ); + if let Some(age) = age { + headers.insert(header::AGE, HeaderValue::from_str(age).expect("age")); + } + if let Some(date_age) = date_age { + let date = httpdate::fmt_http_date( + std::time::UNIX_EPOCH + Duration::from_secs(receipt_seconds - date_age), + ); + headers.insert(header::DATE, HeaderValue::from_str(&date).expect("date")); + } + assert_eq!( + OriginFacts::of_at(&headers, receipt).remaining_freshness(Duration::from_secs(elapsed)), + OriginFreshness { + browser: Duration::from_secs(browser), + shared: Duration::from_secs(shared), + }, + "{control}, Age {age:?}, apparent age {date_age:?}, processing {elapsed}", + ); + } +} + +#[test] +fn invalid_origin_date_cannot_gain_freshness_when_rebased() { + let mut headers = HeaderMap::new(); + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=3600"), + ); + headers.insert(header::DATE, HeaderValue::from_static("not a date")); + assert_eq!( + OriginFacts::of_at(&headers, UNIX_EPOCH).remaining_freshness(Duration::ZERO), + OriginFreshness::default(), + ); +} diff --git a/crates/trusted-server-core/src/middleware.rs b/crates/trusted-server-core/src/middleware.rs index f8555ff0fe..a0a072eb4d 100644 --- a/crates/trusted-server-core/src/middleware.rs +++ b/crates/trusted-server-core/src/middleware.rs @@ -46,7 +46,7 @@ use std::rc::Rc; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Duration; -use http::{HeaderName, HeaderValue, StatusCode}; +use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; use crate::attestation::PlatformIdentity; @@ -59,6 +59,17 @@ use crate::streaming_processor::StreamProcessor; /// unless it says it handles another. pub const HTML_MEDIA_TYPE: &str = "text/html"; +/// A module's exact original byte ranges that no document transform may edit. +/// Called on the complete, bounded document before core or middleware runs. +/// Each classifier returns ordered, disjoint, nonempty ranges inside the input. +/// An error refuses the document. Core treats ranges as opaque; classifications +/// from different middleware are combined, including overlapping ranges. +/// +/// Transforms see an opaque comment in place of each protected range. A +/// middleware that signs or hashes raw whole HTML must not bind that intermediate +/// view and assume its binding remains valid after original bytes are restored. +pub type SourceProtection = fn(&[u8]) -> Result>, std::io::Error>; + /// Whether a middleware may run on a response of `media_type`, written as a /// bare `type/subtype` in lower case. /// @@ -154,6 +165,13 @@ pub struct MiddlewareContext<'a> { /// a reader's own page alone. A page from the store of shared templates /// was never one reader's. pub origin_private: bool, + /// The headers of the reader's response as core will send them, being + /// the origin's after core moved its addresses to the reader's host, for + /// a middleware whose change has to respect a header such as + /// `Content-Security-Policy`. `None` where core cannot say, being the + /// fetch phase of a page that may be stored and a page replayed from the + /// store, which a middleware treats as headers it does not know. + pub response_headers: Option<&'a HeaderMap>, /// What the adapter reports about the platform the deployment runs on, /// as the attestation evidence carries it. pub platform: &'a PlatformIdentity, @@ -567,6 +585,13 @@ pub trait Middleware: Send + Sync { false } + /// Classifies source bytes that must survive all document transformations + /// unchanged. Default off. Only middleware in the applicable chains may + /// request this hold, including serve middleware before a template is built. + fn source_protection(&self) -> Option { + None + } + /// Whether this middleware moves the origin's address to the reader's /// host itself, in a body of a media type it handles other than HTML. /// Core then leaves the addresses in such a body alone, whether or not @@ -927,6 +952,15 @@ impl MiddlewareChain { self.middleware.is_empty() } + /// The original-source classifiers selected by this document's chain. + #[must_use] + pub fn source_protections(&self) -> Vec { + self.middleware + .iter() + .filter_map(|middleware| middleware.source_protection()) + .collect() + } + /// Whether a middleware in the chain writes the whole document, see /// [`Middleware::writes_whole_document`]. #[must_use] @@ -1115,6 +1149,7 @@ pub mod test_support { origin_host: ORIGIN_HOST, status: http::StatusCode::OK, origin_private: false, + response_headers: None, platform, document_state, max_buffered_script_bytes: 16 * 1024 * 1024, diff --git a/crates/trusted-server-core/src/middleware/tests.rs b/crates/trusted-server-core/src/middleware/tests.rs index fc493f74e1..6275024e64 100644 --- a/crates/trusted-server-core/src/middleware/tests.rs +++ b/crates/trusted-server-core/src/middleware/tests.rs @@ -69,6 +69,7 @@ fn reader(request_state: &IntegrationRequestState) -> ReaderDocument<'_> { origin_host: ORIGIN_HOST, status: StatusCode::OK, origin_private: false, + response_headers: None, platform: test_support::platform(), request_state, max_buffered_script_bytes: 16 * 1024 * 1024, @@ -2390,3 +2391,119 @@ fn an_html_entry_ahead_does_not_shadow_the_entry_that_makes_the_file() { "should make the file from the text/plain entry, not as HTML" ); } + +/// A format-agnostic source policy supplied by a test module. Core knows only +/// its returned byte range, not why that subtree must remain unchanged. +struct ProtectedFixture; + +const PROTECTED_FIXTURE: &str = "

keep

keep

"; + +impl Middleware for ProtectedFixture { + fn middleware_id(&self) -> &'static str { + "fixture.protected" + } + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + fn create(&self, _: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } + fn source_protection(&self) -> Option { + Some(|original| { + Ok(original + .windows(PROTECTED_FIXTURE.len()) + .position(|bytes| bytes == PROTECTED_FIXTURE.as_bytes()) + .map(|start| start..start + PROTECTED_FIXTURE.len()) + .into_iter() + .collect()) + }) + } +} + +#[test] +fn source_policy_protects_before_core_fetch_and_warm_serve_descendant_handlers() { + use crate::html_processor::create_html_processor_with_source_protection; + + let fetch = chain_of(vec![ + element("fixture.fetch-attribute", "a.edit", "href", |_| { + AttributeRewriteAction::Replace("fetch-edited".to_owned()) + }), + text("fixture.fetch-text", "p.edit", || { + |_, last| { + if last { + ScriptRewriteAction::Replace("fetch-edited".to_owned()) + } else { + ScriptRewriteAction::remove_node() + } + } + }), + ]); + let serve = serve_chain_of(vec![ + Arc::new(ProtectedFixture), + element("fixture.serve-attribute", "a.edit", "href", |_| { + AttributeRewriteAction::Replace("serve-edited".to_owned()) + }), + text("fixture.serve-text", "p.edit", || { + |_, last| { + if last { + ScriptRewriteAction::Replace("serve-edited".to_owned()) + } else { + ScriptRewriteAction::remove_node() + } + } + }), + ]); + let html = format!( + "{PROTECTED_FIXTURE}\ + outside\ +

outside

" + ); + // The source policy lives in serve, but must already cover fetch/template + // construction, without running any of serve's actions early. + let (mut processor, _) = + create_html_processor_with_source_protection(config(), &fetch, None, &[&fetch, &serve]) + .expect("plan template"); + for chunk in html.as_bytes().chunks(3) { + assert!( + processor + .process_chunk(chunk, false) + .expect("hold original") + .is_empty() + ); + } + let template = String::from_utf8(processor.process_chunk(&[], true).expect("finish template")) + .expect("UTF-8"); + assert!(template.contains(PROTECTED_FIXTURE)); + assert!(template.contains("href=\"fetch-edited\"")); + assert!(template.contains(">fetch-edited

")); + assert!(!template.contains("serve-edited")); + assert!(!template.contains("