From a0a48b7abb3d77049143199c340ffeb0c9b2e539 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 10 Oct 2026 02:43:38 +0100 Subject: [PATCH 1/8] A middleware may run on a media type other than HTML An entry may cover any media type core streams as text: `text/html`, another `text/` type other than `text/x-component`, whose rows are length prefixed, `application/javascript`, `application/json` or `image/svg+xml`, written as a bare `type/subtype` in lower case. A response is matched by the type and subtype of its content type, without the parameters. A middleware says which media types it works on with `handles_media_type`, HTML alone unless it says otherwise, so every module keeps its behavior with no change to its code. An entry naming a middleware under a media type it does not handle is refused when the registry is built, naming the entry and the middleware. A chain carries its media type, and on one other than HTML its plan refuses an action that decides about HTML, being head markup or an element or text handler, by the middleware's name, because those are decisions about HTML and a stream processor is the one thing that applies to any body. The publisher runs the covering entries' stream processors on a body of such a media type after it has moved the origin's addresses in it: the fetch entry's on what is stored, and the serve entry's on each reader's copy, whether that copy is assembled from the store or fetched for the request, as on a page. Core's test fixture gains a middleware that works on `text/plain` and adds a line at the end of a body, and the tests cover the entries accepted and refused, the media type of a content type, a chain refusing an HTML decision on text and taking a stream, the registry refusing a middleware under a media type it does not handle in either direction, and a text body through the publisher with and without a covering entry. --- crates/cmp/inmobi/src/lib.rs | 12 +- .../src/integrations/js_asset_proxy.rs | 1 + .../src/integrations/registry.rs | 151 +++++++- crates/trusted-server-core/src/middleware.rs | 105 +++++- .../src/middleware/tests.rs | 112 +++++- crates/trusted-server-core/src/publisher.rs | 344 +++++++++++++++++- 6 files changed, 675 insertions(+), 50 deletions(-) diff --git a/crates/cmp/inmobi/src/lib.rs b/crates/cmp/inmobi/src/lib.rs index 81c158a53c..ae6373a1fb 100644 --- a/crates/cmp/inmobi/src/lib.rs +++ b/crates/cmp/inmobi/src/lib.rs @@ -316,10 +316,14 @@ mod tests { /// The head markup `middleware` write in `phase`, in the order written. fn head_markup(phase: MiddlewarePhase, middleware: Vec>) -> Vec { let document_state = IntegrationDocumentState::default(); - MiddlewareChain::new(phase, middleware) - .plan(&context(phase, &document_state)) - .expect("the chain should plan") - .head_inserts + MiddlewareChain::new( + phase, + trusted_server_core::middleware::HTML_MEDIA_TYPE, + middleware, + ) + .plan(&context(phase, &document_state)) + .expect("the chain should plan") + .head_inserts } /// The one piece of head markup the prompt writes in `phase`. diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs index 2872a114e6..a8b8a38b0b 100644 --- a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -674,6 +674,7 @@ mod tests { ) -> String { let chain = MiddlewareChain::new( MiddlewarePhase::Fetch, + crate::middleware::HTML_MEDIA_TYPE, vec![Arc::new(ScriptAddress(integration))], ); let config = HtmlProcessorConfig { diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index fb37af1a1e..b29039819e 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1407,6 +1407,13 @@ fn check_middleware( phases_of(middleware).join(" and ") ))); } + if !middleware.handles_media_type(&entry.media_type) { + return Err(refuse(format!( + "{at} covers `{}` and names `{name}`, which does not work on that media \ + type", + entry.media_type + ))); + } } } } @@ -1917,7 +1924,7 @@ impl IntegrationRegistry { .iter() .filter_map(|name| self.inner.middleware_named(name).map(Arc::clone)) .collect(); - MiddlewareChain::new(phase, middleware) + MiddlewareChain::new(phase, media_type, middleware) } /// Every integration id the registry was built from, named or not, in @@ -2924,6 +2931,12 @@ pub(crate) mod test_support { pub(crate) const LINK_TARGET: &str = "/fixture/link"; /// The selector the broken middleware asks for. pub(crate) const BROKEN_SELECTOR: &str = "a["; + /// The name of the fetch middleware that works on `text/plain`. + pub(crate) const TEXT: &str = "testing.middleware-fixture.text"; + /// The media type the text middleware works on. + pub(crate) const TEXT_MEDIA_TYPE: &str = "text/plain"; + /// The line the text middleware adds at the end of a text body. + pub(crate) const TEXT_LINE: &str = "# added by the middleware fixture\n"; /// The builder core's test build lists beside its own. pub(crate) const BUILDER: IntegrationBuilder = @@ -3045,6 +3058,46 @@ pub(crate) mod test_support { } } + /// Passes a text body through and adds a line at its end. + struct AppendLine; + + impl crate::streaming_processor::StreamProcessor for AppendLine { + fn process_chunk( + &mut self, + chunk: &[u8], + is_last: bool, + ) -> Result, std::io::Error> { + let mut output = chunk.to_vec(); + if is_last { + output.extend_from_slice(TEXT_LINE.as_bytes()); + } + Ok(output) + } + } + + struct Text; + + impl Middleware for Text { + fn middleware_id(&self) -> &'static str { + TEXT + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == TEXT_MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + stream: Some(Box::new(AppendLine)), + ..MiddlewareAction::pass() + } + } + } + struct Broken; impl Middleware for Broken { @@ -3084,6 +3137,7 @@ pub(crate) mod test_support { .with_middleware(Arc::new(Broken)) .with_middleware(Arc::new(Reader)) .with_middleware(Arc::new(BrokenReader)) + .with_middleware(Arc::new(Text)) .build(), )) } @@ -5989,12 +6043,18 @@ mod tests { fixture::BROKEN, fixture::READER, fixture::BROKEN_READER, + fixture::TEXT, ], "should list the middleware the selected module supplies" ); assert_eq!( registry.middleware_in(MiddlewarePhase::Fetch), - [fixture::HEAD, fixture::LINKS, fixture::BROKEN], + [ + fixture::HEAD, + fixture::LINKS, + fixture::BROKEN, + fixture::TEXT + ], "should list the middleware that run in the fetch phase" ); assert_eq!( @@ -6019,6 +6079,73 @@ mod tests { ); } + /// An entry for `text/plain` naming the middleware that works on it. + fn text_entry(names: &[&str]) -> PhaseEntry { + PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: names.iter().map(|name| (*name).to_owned()).collect(), + } + } + + #[test] + fn an_entry_runs_a_middleware_on_the_media_type_it_handles() { + let settings = fixture_settings_with_entries(vec![ + html_entry(None, &[fixture::HEAD]), + text_entry(&[fixture::TEXT]), + ]); + + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + + let chain_for = |media_type: &str| { + registry + .middleware_chain( + &settings.fetch, + MiddlewarePhase::Fetch, + media_type, + "/a.txt", + ) + .ids() + }; + assert_eq!( + chain_for(fixture::TEXT_MEDIA_TYPE), + [fixture::TEXT], + "should run the text middleware on a text response" + ); + assert_eq!( + chain_for(HTML_MEDIA_TYPE), + [fixture::HEAD], + "should run the page middleware on a page and not the text one" + ); + } + + #[test] + fn an_entry_naming_a_middleware_under_a_media_type_it_does_not_handle_is_refused() { + for (media_type, name) in [ + (fixture::TEXT_MEDIA_TYPE, fixture::HEAD), + (HTML_MEDIA_TYPE, fixture::TEXT), + ] { + let settings = fixture_settings_with_entries(vec![PhaseEntry { + media_type: media_type.to_owned(), + path: None, + middleware: vec![name.to_owned()], + }]); + + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse a middleware under a media type it does not handle"); + + let message = error.to_string(); + assert!( + message.contains(&format!( + "[[fetch]] entry 1 covers `{media_type}` and names `{name}`, which does not \ + work on that media type" + )), + "should name the entry, the media type and the middleware: {message}" + ); + } + } + #[test] fn a_module_no_section_selects_supplies_no_middleware() { let settings = crate::test_support::tests::create_test_settings(); @@ -6047,12 +6174,13 @@ mod tests { message.contains( "[[fetch]] entry 2 names `testing.nothing`, which no module that runs supplies" ) && message.contains(&format!( - "[{}, {}, {}, {}, {}]", + "[{}, {}, {}, {}, {}, {}]", fixture::HEAD, fixture::LINKS, fixture::BROKEN, fixture::READER, - fixture::BROKEN_READER + fixture::BROKEN_READER, + fixture::TEXT )), "should name the entry and list what could be named: {message}" ); @@ -6104,6 +6232,7 @@ mod tests { fixture::BROKEN, fixture::READER, fixture::BROKEN_READER, + fixture::TEXT, ], "should list every middleware when there are no entries" ); @@ -6112,7 +6241,12 @@ mod tests { html_entry(Some("/news/"), &[fixture::LINKS]), html_entry(None, &[fixture::HEAD]), ]), - [fixture::BROKEN, fixture::READER, fixture::BROKEN_READER], + [ + fixture::BROKEN, + fixture::READER, + fixture::BROKEN_READER, + fixture::TEXT + ], "should leave out a middleware any entry names" ); } @@ -6175,7 +6309,12 @@ mod tests { .collect(); assert_eq!( unnamed, - [fixture::LINKS, fixture::BROKEN, fixture::BROKEN_READER], + [ + fixture::LINKS, + fixture::BROKEN, + fixture::BROKEN_READER, + fixture::TEXT + ], "should count a middleware as named only by an entry of a phase it runs in" ); } diff --git a/crates/trusted-server-core/src/middleware.rs b/crates/trusted-server-core/src/middleware.rs index bd9c07867e..6c46bef516 100644 --- a/crates/trusted-server-core/src/middleware.rs +++ b/crates/trusted-server-core/src/middleware.rs @@ -44,10 +44,39 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; use crate::integrations::{AttributeRewriteAction, IntegrationDocumentState, ScriptRewriteAction}; use crate::streaming_processor::StreamProcessor; -/// The media type a document is matched as, and the only one a middleware -/// runs on. +/// The media type a document is matched as, and the one a middleware runs on +/// unless it says it handles another. pub const HTML_MEDIA_TYPE: &str = "text/html"; +/// Whether a middleware may run on a response of `media_type`, written as a +/// bare `type/subtype` in lower case. +/// +/// HTML and the text assets that stream through core as text: every `text/` +/// type except `text/x-component`, whose rows are length prefixed, and +/// JavaScript, JSON and SVG. Anything else passes through untouched. +#[must_use] +pub fn is_dispatched_media_type(media_type: &str) -> bool { + media_type != "text/x-component" + && (media_type.starts_with("text/") + || matches!( + media_type, + "application/javascript" | "application/json" | "image/svg+xml" + )) +} + +/// The type and subtype of a content type header, in lower case and without +/// parameters, which is what an entry's `media_type` is matched against. +/// `text/html; charset=utf-8` gives `text/html`. +#[must_use] +pub fn media_type_of(content_type: &str) -> String { + content_type + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase() +} + /// When a middleware runs, relative to the store of shared templates. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum MiddlewarePhase { @@ -241,11 +270,17 @@ impl MiddlewareAction { /// Whether the action leaves the document as it arrived. #[must_use] pub fn is_pass(&self) -> bool { + self.is_stream_only() && self.stream.is_none() + } + + /// Whether the action decides nothing about HTML, so it can run on a + /// media type that is not HTML. + #[must_use] + pub fn is_stream_only(&self) -> bool { self.head_inserts.is_empty() && self.after_bundle_inserts.is_empty() && self.element_handlers.is_empty() && self.text_handlers.is_empty() - && self.stream.is_none() } /// Checks every handler's selector parses, before a rewriter is built @@ -309,6 +344,16 @@ pub trait Middleware: Send + Sync { /// names its own by a bare word. fn middleware_id(&self) -> &'static str; + /// Whether this middleware works on a response of `media_type`, given as + /// a bare `type/subtype`. HTML alone unless a middleware says otherwise, + /// and an entry naming it under a media type it does not handle is + /// refused when the registry is built. On a media type other than HTML + /// an action's stream processor alone runs, because head markup and the + /// element and text handlers are decisions about HTML. + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == HTML_MEDIA_TYPE + } + /// The phases an entry may name this middleware in. fn phases(&self) -> &[MiddlewarePhase]; @@ -406,10 +451,15 @@ impl PhaseEntries { pub fn validate(&self, phase: MiddlewarePhase) -> Result<(), String> { for (index, entry) in self.0.iter().enumerate() { let at = format!("[[{phase}]] entry {}", index + 1); - if entry.media_type != HTML_MEDIA_TYPE { + if entry.media_type != entry.media_type.to_ascii_lowercase() + || entry.media_type.contains([';', ' ', '*']) + || !is_dispatched_media_type(&entry.media_type) + { return Err(format!( - "{at} covers `{}`, and a middleware runs on `{HTML_MEDIA_TYPE}` alone. Write \ - media_type = \"{HTML_MEDIA_TYPE}\", in lower case with no parameters", + "{at} covers `{}`, which is not a media type a middleware runs on. Write a \ + bare `type/subtype` in lower case with no parameters, being `text/html`, \ + another `text/` type other than `text/x-component`, \ + `application/javascript`, `application/json` or `image/svg+xml`", entry.media_type )); } @@ -498,10 +548,12 @@ impl Serialize for PhaseEntries { } } -/// The middleware one entry selects, in the order they run. +/// The middleware one entry selects, in the order they run, for one media +/// type. #[derive(Clone)] pub struct MiddlewareChain { phase: MiddlewarePhase, + media_type: String, middleware: Vec>, } @@ -510,16 +562,26 @@ impl fmt::Debug for MiddlewareChain { formatter .debug_struct("MiddlewareChain") .field("phase", &self.phase) + .field("media_type", &self.media_type) .field("middleware", &self.ids()) .finish() } } impl MiddlewareChain { - /// A chain of middleware already in the order they run. + /// A chain of middleware already in the order they run, for responses of + /// `media_type`. #[must_use] - pub fn new(phase: MiddlewarePhase, middleware: Vec>) -> Self { - Self { phase, middleware } + pub fn new( + phase: MiddlewarePhase, + media_type: &str, + middleware: Vec>, + ) -> Self { + Self { + phase, + media_type: media_type.to_owned(), + middleware, + } } /// The phase this chain runs in. @@ -528,6 +590,12 @@ impl MiddlewareChain { self.phase } + /// The media type of the responses this chain runs on. + #[must_use] + pub fn media_type(&self) -> &str { + &self.media_type + } + /// The middleware names, in the order they run. #[must_use] pub fn ids(&self) -> Vec<&'static str> { @@ -549,11 +617,14 @@ impl MiddlewareChain { /// middleware chooses its handlers for each document, so each is checked /// here. The caller refuses the response when one does not parse, because /// a change that silently does not happen is worse than a response that - /// fails. + /// fails. On a media type other than HTML an action may carry a stream + /// processor alone, and one that decides about HTML is refused the same + /// way. /// /// # Errors /// - /// A message naming the middleware and the selector that does not parse. + /// A message naming the middleware and the selector that does not parse, + /// or the HTML decision it made about a document that is not HTML. pub fn plan(&self, context: &MiddlewareContext<'_>) -> Result { let mut plan = MiddlewarePlan::default(); for middleware in &self.middleware { @@ -565,6 +636,16 @@ impl MiddlewareChain { middleware.middleware_id() ) })?; + if self.media_type != HTML_MEDIA_TYPE && !action.is_stream_only() { + return Err(format!( + "the {} middleware `{}` cannot run on `{}`, because it decides about HTML \ + and that is not HTML. On another media type an action carries a stream \ + processor alone", + self.phase, + middleware.middleware_id(), + self.media_type + )); + } plan.head_inserts.extend(action.head_inserts); plan.after_bundle_inserts .extend(action.after_bundle_inserts); diff --git a/crates/trusted-server-core/src/middleware/tests.rs b/crates/trusted-server-core/src/middleware/tests.rs index d084dcca2e..2f33d32e68 100644 --- a/crates/trusted-server-core/src/middleware/tests.rs +++ b/crates/trusted-server-core/src/middleware/tests.rs @@ -11,7 +11,7 @@ use crate::html_processor::{ create_html_processor_with_middleware, create_serve_processor, }; use crate::integrations::registry_test_support::request_fixture; -use crate::integrations::{IntegrationRegistry, IntegrationRequestState}; +use crate::integrations::{IntegrationDocumentState, IntegrationRegistry, IntegrationRequestState}; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use crate::test_support::tests::create_test_settings; @@ -54,11 +54,11 @@ fn middleware( } fn chain_of(middleware: Vec>) -> MiddlewareChain { - MiddlewareChain::new(MiddlewarePhase::Fetch, middleware) + MiddlewareChain::new(MiddlewarePhase::Fetch, HTML_MEDIA_TYPE, middleware) } fn serve_chain_of(middleware: Vec>) -> MiddlewareChain { - MiddlewareChain::new(MiddlewarePhase::Serve, middleware) + MiddlewareChain::new(MiddlewarePhase::Serve, HTML_MEDIA_TYPE, middleware) } /// What a serve chain is told of a reader whose request left `request_state`. @@ -311,22 +311,108 @@ fn entries_from_the_longest_path_to_every_path_are_accepted() { } #[test] -fn an_entry_for_a_media_type_other_than_html_is_refused() { - for media_type in ["text/css", "text/html; charset=utf-8", "TEXT/HTML", "*/*"] { - let mut css = entry(None, &["example.a"]); - css.media_type = media_type.to_owned(); - - let message = refusal(vec![css]); +fn an_entry_for_a_media_type_a_middleware_does_not_run_on_is_refused() { + for media_type in [ + "text/html; charset=utf-8", + "TEXT/HTML", + "text/PLAIN", + "*/*", + "text/*", + "image/png", + "application/octet-stream", + "text/x-component", + ] { + let mut other = entry(None, &["example.a"]); + other.media_type = media_type.to_owned(); + + let message = refusal(vec![other]); assert!( message.contains("[[fetch]] entry 1") && message.contains(&format!("covers `{media_type}`")) - && message.contains("media_type = \"text/html\""), - "should name the entry and the media type to write: {message}" + && message.contains("is not a media type a middleware runs on") + && message.contains("`text/html`"), + "should name the entry, the media type and what to write: {message}" ); } } +#[test] +fn an_entry_may_cover_a_text_media_type_that_is_not_html() { + for media_type in [ + "text/plain", + "text/css", + "application/javascript", + "application/json", + "image/svg+xml", + ] { + let mut text = entry(None, &["example.a"]); + text.media_type = media_type.to_owned(); + + PhaseEntries::new(vec![text]) + .validate(MiddlewarePhase::Fetch) + .unwrap_or_else(|message| panic!("{media_type} should be accepted: {message}")); + } +} + +#[test] +fn a_media_type_is_the_type_and_subtype_of_a_content_type_in_lower_case() { + assert_eq!(media_type_of("text/html; charset=utf-8"), "text/html"); + assert_eq!(media_type_of(" Text/Plain "), "text/plain"); + assert_eq!(media_type_of("application/json"), "application/json"); + assert_eq!(media_type_of(""), ""); + assert!(is_dispatched_media_type("text/plain")); + assert!(is_dispatched_media_type("image/svg+xml")); + assert!( + !is_dispatched_media_type("text/x-component"), + "a length prefixed stream is not changed" + ); + assert!(!is_dispatched_media_type("image/png")); +} + +#[test] +fn a_chain_on_another_media_type_runs_a_stream_processor_alone() { + let document_state = IntegrationDocumentState::default(); + let context = super::test_support::context(MiddlewarePhase::Fetch, &document_state); + let head = middleware("example.head", |_context| MiddlewareAction { + head_inserts: vec!["".to_owned()], + ..MiddlewareAction::pass() + }); + let stream = middleware("example.stream", |_context| MiddlewareAction { + stream: Some(Box::new(Append("# more".to_owned()))), + ..MiddlewareAction::pass() + }); + + let refused = MiddlewareChain::new(MiddlewarePhase::Fetch, "text/plain", vec![head]) + .plan(&context) + .expect_err("a head insert is a decision about HTML"); + assert!( + refused.contains("`example.head`") + && refused.contains("cannot run on `text/plain`") + && refused.contains("decides about HTML"), + "should name the middleware, the media type and the reason: {refused}" + ); + + let plan = MiddlewareChain::new(MiddlewarePhase::Fetch, "text/plain", vec![stream]) + .plan(&context) + .expect("a stream processor runs on any media type"); + assert_eq!(plan.processors.len(), 1); + assert!(plan.head_inserts.is_empty()); +} + +/// A stream processor that adds `text` at the end of a body. +struct Append(String); + +impl crate::streaming_processor::StreamProcessor for Append { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error> { + let mut output = chunk.to_vec(); + if is_last { + output.extend_from_slice(self.0.as_bytes()); + } + Ok(output) + } +} + #[test] fn an_entry_whose_path_could_cover_nothing_is_refused() { let message = refusal(vec![entry(Some("news/"), &["example.a"])]); @@ -1368,8 +1454,8 @@ fn settings_without_entries_write_no_fetch_key() { fn settings_with_a_misshapen_entry_are_refused() { let cases = [ ( - "\n[[fetch]]\nmedia_type = \"text/css\"\nmiddleware = [\"example.all\"]\n", - "[[fetch]] entry 1 covers `text/css`", + "\n[[fetch]]\nmedia_type = \"image/png\"\nmiddleware = [\"example.all\"]\n", + "[[fetch]] entry 1 covers `image/png`", ), ( "\n[fetch]\nmedia_type = \"text/html\"\nmiddleware = [\"example.all\"]\n", diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index c9ac107eff..7c5a0eded9 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -68,7 +68,8 @@ use crate::ec::{EcContext, EidSyncSource}; use crate::error::TrustedServerError; use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; -use crate::integrations::{IntegrationRegistry, IntegrationRequestState}; +use crate::integrations::{IntegrationDocumentState, IntegrationRegistry, IntegrationRequestState}; +use crate::middleware::{MiddlewarePhase, media_type_of}; use crate::permissions::PermissionState; use crate::platform::{ GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, @@ -652,6 +653,95 @@ struct PublisherBodyProcessor { inner: Box, } +/// A response that is not HTML, as the middleware the entries covering it +/// are told of it. +struct TextDocument<'a> { + /// The type and subtype of the response, in lower case. + media_type: String, + request_path: &'a str, + request_host: &'a str, + request_scheme: &'a str, + origin_host: &'a str, + request_state: &'a IntegrationRequestState, + max_buffered_script_bytes: usize, +} + +/// The stream processors the entries covering a response that is not HTML +/// run on its body, phase by phase in the order given, or none when its +/// media type is not one a middleware runs on or no entry covers it. +/// +/// A fetch middleware is told nothing a request left, as on a page, and a +/// serve middleware is told what this reader's request left. +/// +/// # Errors +/// +/// When a middleware decides about HTML on a document that is not HTML, or +/// asks for a selector that does not parse. +fn middleware_stream_processors( + document: &TextDocument<'_>, + phases: &[MiddlewarePhase], + settings: &Settings, + integration_registry: &IntegrationRegistry, +) -> Result>, Report> { + use crate::middleware::{MiddlewareContext, is_dispatched_media_type}; + + if !is_dispatched_media_type(&document.media_type) { + return Ok(Vec::new()); + } + let mut processors = Vec::new(); + for phase in phases { + let chain = integration_registry.middleware_chain( + settings.phase_entries(*phase), + *phase, + &document.media_type, + document.request_path, + ); + if chain.is_empty() { + continue; + } + let document_state = IntegrationDocumentState::default(); + if *phase == MiddlewarePhase::Serve { + document.request_state.seed(&document_state); + } + let plan = chain + .plan(&MiddlewareContext { + phase: *phase, + request_host: document.request_host, + request_scheme: document.request_scheme, + origin_host: document.origin_host, + document_state: &document_state, + max_buffered_script_bytes: document.max_buffered_script_bytes, + }) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; + processors.extend(plan.processors); + } + Ok(processors) +} + +/// A processor followed by the ones the entries add, each handed what the +/// one before produced. +struct WithStreamProcessors { + inner: Box, + processors: Vec>, +} + +impl StreamProcessor for WithStreamProcessors { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, std::io::Error> { + let mut output = self.inner.process_chunk(chunk, is_last)?; + for processor in &mut self.processors { + output = processor.process_chunk(&output, is_last)?; + } + Ok(output) + } + + fn reset(&mut self) { + self.inner.reset(); + for processor in &mut self.processors { + processor.reset(); + } + } +} + impl PublisherBodyProcessor { /// Build the body processor, returning any deferred inline seam token it /// installed alongside it. @@ -699,12 +789,37 @@ impl PublisherBodyProcessor { ¶ms.request_scheme, )) } else { - Box::new(create_url_replacer( - ¶ms.origin_host, - ¶ms.origin_url, - ¶ms.request_host, - ¶ms.request_scheme, - )) + // The serve entry runs on each reader's copy of a stored response + // later, see `reader_serve_processor`, and here on one that is not + // stored. + let phases: &[MiddlewarePhase] = if params.template_cache_key.is_some() { + &[MiddlewarePhase::Fetch] + } else { + &MiddlewarePhase::ALL + }; + let processors = middleware_stream_processors( + &TextDocument { + media_type: media_type_of(¶ms.content_type), + request_path: ¶ms.request_path, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + origin_host: ¶ms.origin_host, + request_state: ¶ms.request_state, + max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, + }, + phases, + settings, + integration_registry, + )?; + Box::new(WithStreamProcessors { + inner: Box::new(create_url_replacer( + ¶ms.origin_host, + ¶ms.origin_url, + ¶ms.request_host, + ¶ms.request_scheme, + )), + processors, + }) }; Ok((Self { inner }, inline_seam_token)) @@ -790,13 +905,35 @@ fn process_response_streaming( .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; } else { - let replacer = create_url_replacer( - params.origin_host, - params.origin_url, - params.request_host, - params.request_scheme, - ); - StreamingPipeline::new(config, replacer) + let phases: &[MiddlewarePhase] = if params.shared_template_authorized { + &[MiddlewarePhase::Fetch] + } else { + &MiddlewarePhase::ALL + }; + let processors = middleware_stream_processors( + &TextDocument { + media_type: media_type_of(params.content_type), + request_path: params.request_path, + request_host: params.request_host, + request_scheme: params.request_scheme, + origin_host: params.origin_host, + request_state: params.request_state, + max_buffered_script_bytes: params.settings.publisher.max_buffered_body_bytes, + }, + phases, + params.settings, + params.integration_registry, + )?; + let processor = WithStreamProcessors { + inner: Box::new(create_url_replacer( + params.origin_host, + params.origin_url, + params.request_host, + params.request_scheme, + )), + processors, + }; + StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; } @@ -1575,7 +1712,29 @@ fn reader_serve_processor( use crate::middleware::{HTML_MEDIA_TYPE, MiddlewarePhase}; if !is_html_content_type(¶ms.content_type) { - return Ok(None); + let origin_host = settings.publisher.origin_host(); + let mut processors = middleware_stream_processors( + &TextDocument { + media_type: media_type_of(¶ms.content_type), + request_path: ¶ms.request_path, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + origin_host: &origin_host, + request_state: ¶ms.request_state, + max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, + }, + &[MiddlewarePhase::Serve], + settings, + integration_registry, + )?; + return Ok(match processors.len() { + 0 => None, + 1 => processors.pop(), + _ => Some(Box::new(WithStreamProcessors { + inner: processors.remove(0), + processors, + })), + }); } let chain = integration_registry.middleware_chain( &settings.serve, @@ -7780,6 +7939,7 @@ mod tests { use crate::integrations::{ CarriedJsModule, IntegrationBuilder, IntegrationRegistration, IntegrationRegistry, }; + use crate::middleware::{PhaseEntries, PhaseEntry}; use crate::permissions::{Permission, PermissionSet}; use crate::platform::test_support::{ StubHttpClient, build_services_with_http_client, noop_services, @@ -20583,6 +20743,160 @@ mod tests { )) } + /// The settings of a deployment running the middleware fixture, with the + /// `[[fetch]]` entry given. + fn settings_running_the_middleware_fixture(fetch: Vec) -> Settings { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let mut settings = create_test_settings(); + settings.select_module("testing", fixture::MODULE); + settings.fetch = PhaseEntries::new(fetch); + settings + } + + /// A text body streamed through the publisher path, as a reader gets it. + fn text_body_through_the_publisher(settings: &Settings, body: &str) -> String { + let registry = IntegrationRegistry::new(settings).expect("should build a registry"); + let params = OwnedProcessResponseParams { + csp_nonce_observed: None, + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/a.txt".to_owned(), + content_type: "text/plain; charset=utf-8".to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }; + let mut output = Vec::new(); + stream_publisher_body( + EdgeBody::from(body.as_bytes().to_vec()), + &mut output, + ¶ms, + settings, + ®istry, + ) + .expect("should stream a text body"); + String::from_utf8(output).expect("text") + } + + #[test] + fn a_text_body_passes_through_the_middleware_of_the_entry_covering_it() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT.to_owned()], + }]); + let body = "User-agent: *\nSee https://origin.example.com/more\n"; + + let output = text_body_through_the_publisher(&settings, body); + + assert_eq!( + output, + format!( + "User-agent: *\nSee https://proxy.example.com/more\n{}", + fixture::TEXT_LINE + ), + "should move the origin's address and then run the entry's middleware" + ); + } + + /// The same through the response path a streamed body takes, which + /// builds its processor apart from the buffered path's. + #[tokio::test] + async fn a_streamed_text_body_passes_through_the_middleware_of_the_entry_covering_it() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::platform::test_support::noop_services; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT.to_owned()], + }]); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + let services = noop_services(); + let mut params = OwnedProcessResponseParams { + csp_nonce_observed: None, + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/a.txt".to_owned(), + content_type: "text/plain; charset=utf-8".to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }; + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"User-agent: *\nSee https://origin.example.com/"), + bytes::Bytes::from_static(b"more\n"), + ])); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("should stream a text body"); + + assert_eq!( + String::from_utf8(output).expect("text"), + format!( + "User-agent: *\nSee https://proxy.example.com/more\n{}", + fixture::TEXT_LINE + ), + "should move the origin's address and then run the entry's middleware" + ); + } + + #[test] + fn a_text_body_no_entry_covers_is_left_as_it_was() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: Some("/elsewhere/".to_owned()), + middleware: vec![fixture::TEXT.to_owned()], + }]); + + let output = text_body_through_the_publisher(&settings, "plain\n"); + + assert_eq!( + output, "plain\n", + "should run nothing on a path the entry does not cover" + ); + } + #[test] fn tsjs_dynamic_serves_a_standalone_module_by_registration_flag_not_by_name() { // The module the registration marks standalone is one a vendor crate From 218636c41c5c93b7dc15ae450aaae7c6500e6154 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 10 Oct 2026 02:43:39 +0100 Subject: [PATCH 2/8] The guides describe entries for other media types The configuration guide says which media types an entry may cover, how a response is matched, what runs on a body that is not HTML and what is refused, with an entry for a text file as the example. The integration guide shows `handles_media_type` on a middleware that works on `text/plain`. The changelog records the change. --- CHANGELOG.md | 1 + docs/guide/configuration.md | 30 ++++++++++++++++++++++++- docs/guide/integration-guide.md | 40 +++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4730e874ad..f95bd21a5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- A `[[fetch]]` or `[[serve]]` entry may cover a media type other than `text/html`: another `text/` type except `text/x-component`, `application/javascript`, `application/json` or `image/svg+xml`, written as a bare `type/subtype` in lower case. A middleware says which media types it works on with `handles_media_type`, HTML alone unless it says otherwise, and an entry naming one under a media type it does not handle is refused when the registry is built. On a media type other than HTML a middleware's stream processor alone runs on the body, after core has moved the origin's addresses, in the fetch phase and then, for a response that is not stored, the serve phase, and an action that decides about HTML there is refused by name. - The InMobi Choice consent prompt as a module, `[cmp] module = "inmobi"` with `[cmp.inmobi] script_url`, from `crates/cmp/inmobi`. Where a `[[fetch]]` entry names `cmp.inmobi`, its middleware writes the IAB TCF stub, InMobi's GPP stub and the deferred Choice loader into the head, in that order, and the two stubs' behavior is tested with Node's test runner, which CI now runs for every module's `tests/*.test.mjs`. The script URL is refused unless it is `https` on an InMobi Choice host with no user name or password, and a selected prompt that no entry places is refused. 51Degrees maintain the crate until InMobi adopt it. - `[robots-txt]` serves a publisher's `/robots.txt` from the settings. `modules` names what makes the file, in order. `refuse_all` and `allow_all` are Trusted Server's own, and any other name is a module that contributes rules through the new `RobotsTxtContributor` extension point, declared with `with_robots_txt_contributor`. Trusted Server writes the file, so a contributor cannot produce a malformed one, a refusal cannot be loosened, and a path in `always_allow` stays open to every crawler refused the whole site. With `refuse_all` selected every response an adapter finalizes carries `X-Robots-Tag: noindex, nofollow`. A module's answer is held in the key-value store until its refresh is due, a module that cannot answer has its last answer served, and with none held the answer is `503` and never an empty file. Without the section `/robots.txt` reaches the publisher's origin as before. - `ts keys rotate` and `ts keys deactivate` rotate and retire the request signing keys from an operator's machine. They write the two stores the service links as `jwks_store` and `signing_keys` through the `fastly` CLI, with the token from `FASTLY_API_TOKEN`. A private key reaches `fastly` on its standard input and never in its arguments, and a key id no platform can store is refused before anything is written. A command that cannot read the config store writes nothing, and a delete that failed part way can be run again. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 65812e8c4b..7640bf55cd 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1695,7 +1695,7 @@ An entry of either list has the same three fields. | Field | Type | Required | Description | | ------------ | ---------------- | -------- | -------------------------------------------------------------------------------------- | -| `media_type` | String | Yes | The media type the entry covers. Only `text/html` is accepted | +| `media_type` | String | Yes | The media type the entry covers, a bare `type/subtype` in lower case, see below | | `path` | String | No | A prefix of the request path, compared as written. Absent, the entry covers every path | | `middleware` | Array of strings | Yes | The middleware to run, in order, each handed the page as the one before it left it | @@ -1734,6 +1734,34 @@ script bundle, after the fetch middleware's. A middleware takes its settings from its module's own `[
.]` table. An entry carries none and switches nothing on. +### Entries for a media type other than HTML + +An entry covers `text/html`, another `text/` type other than +`text/x-component`, `application/javascript`, `application/json` or +`image/svg+xml`, written as a bare `type/subtype` in lower case with no +parameters. A response is matched by the type and subtype of its +`Content-Type`, without the parameters, so `text/plain; charset=utf-8` +takes the entry for `text/plain`. Anything else, an image or a length +prefixed stream among them, passes through untouched and no entry can +cover it. + +A middleware says which media types it works on, HTML alone unless it says +otherwise, and an entry naming one under a media type it does not handle is +refused when the settings load, with the entry and the middleware named. On +a media type other than HTML a middleware changes the body as a stream, +after core has moved the origin's addresses in it, in the order the entry +names them. Head markup and the element and text handlers are decisions +about HTML, and a middleware that makes one there is refused by name. The +fetch entry's middleware run once on what is stored, and the serve entry's +on each reader's copy, as on a page. + +```toml +[[fetch]] +media_type = "text/plain" +path = "/ads.txt" +middleware = ["example.sellers"] +``` + The modules this repository ships supply the middleware below. A deployment that runs several of them names them all in one entry's list, because a page takes one entry, and the order written is the order they run in. The table diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index d8a7cb2a0a..7a49363ec6 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -197,6 +197,46 @@ Production adapters replace every unavailable or fixture service with the target implementation. The builder deliberately panics when a required service is omitted, so adapter startup must construct the complete service graph. +### A middleware for a media type other than HTML + +A middleware works on `text/html` unless it says otherwise, with +`handles_media_type`, and an entry naming it under a media type it does not +handle is refused when the registry is built. On a media type other than +HTML an action may carry a stream processor alone, which is handed the body +after core has moved the origin's addresses in it, chunk by chunk, with the +last chunk marked. Head markup and the element and text handlers are +decisions about HTML, and an action that makes one on another media type is +refused by name when the chain is planned. + +```rust +struct Sellers; + +impl Middleware for Sellers { + fn middleware_id(&self) -> &'static str { + "example.sellers" + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == "text/plain" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + stream: Some(Box::new(AppendLines::new(["example-ssp.com, 1, DIRECT"]))), + ..MiddlewareAction::pass() + } + } +} +``` + +The entry that places it names the media type and, where the change is for +one file, its path. See "Entries for a media type other than HTML" in the +[configuration guide](/guide/configuration#entries-for-a-media-type-other-than-html). + ## Proxy implementation rules An `IntegrationProxy::handle` implementation receives the request and its From 41a410e85f7fbeb07bd9e90c847dd8f3826aeb1e Mon Sep 17 00:00:00 2001 From: James Rogers Date: Mon, 12 Oct 2026 15:18:09 +0100 Subject: [PATCH 3/8] A middleware runs only on a UTF-8 text body it can change An entry may no longer cover image/svg+xml, which core passes through before any middleware runs, and an entry whose media type has an empty subtype or whitespace is refused. A response that declares a charset other than UTF-8 is handed to no middleware, so a UTF-8 append cannot corrupt it. The stored-response branches for a body that is not HTML are gone, because the template cache stores HTML alone, so both phases run as the response is fetched. TextDocument is TextResponse, and the guide and comments say each rule once. --- CHANGELOG.md | 2 +- crates/trusted-server-core/src/middleware.rs | 38 ++++-- .../src/middleware/tests.rs | 22 ++- crates/trusted-server-core/src/publisher.rs | 126 ++++++++---------- docs/guide/configuration.md | 24 ++-- 5 files changed, 116 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 427cbe9a30..23a4d86fb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,7 +78,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- A `[[fetch]]` or `[[serve]]` entry may cover a media type other than `text/html`: another `text/` type except `text/x-component`, `application/javascript`, `application/json` or `image/svg+xml`, written as a bare `type/subtype` in lower case. A middleware says which media types it works on with `handles_media_type`, HTML alone unless it says otherwise, and an entry naming one under a media type it does not handle is refused when the registry is built. On a media type other than HTML a middleware's stream processor alone runs on the body, after core has moved the origin's addresses, in the fetch phase and then, for a response that is not stored, the serve phase, and an action that decides about HTML there is refused by name. +- A `[[fetch]]` or `[[serve]]` entry may cover a media type other than `text/html`: another `text/` type except `text/x-component`, `application/javascript` or `application/json`, written as a bare `type/subtype` in lower case. A response that declares a charset other than UTF-8 passes through untouched. A middleware says which media types it works on with `handles_media_type`, HTML alone unless it says otherwise, and an entry naming one under a media type it does not handle is refused when the registry is built. On a media type other than HTML a middleware's stream processor alone runs on the body, after core has moved the origin's addresses, in the fetch phase and then the serve phase, and an action that decides about HTML there is refused by name. - The InMobi Choice consent prompt as a module, `[cmp] module = "inmobi"` with `[cmp.inmobi] script_url`, from `crates/cmp/inmobi`. Where a `[[fetch]]` entry names `cmp.inmobi`, its middleware writes the IAB TCF stub, InMobi's GPP stub and the deferred Choice loader into the head, in that order, and the two stubs' behavior is tested with Node's test runner, which CI now runs for every module's `tests/*.test.mjs`. The script URL is refused unless it is `https` on an InMobi Choice host with no user name or password, and a selected prompt that no entry places is refused. The Trusted Server maintainers hold the crate until InMobi adopt it. - `[robots-txt]` serves a publisher's `/robots.txt` from the settings. `modules` names what makes the file, in order. `refuse_all` and `allow_all` are Trusted Server's own, and any other name is a module that contributes rules through the new `RobotsTxtContributor` extension point, declared with `with_robots_txt_contributor`. Trusted Server writes the file, so a contributor cannot produce a malformed one, a refusal cannot be loosened, and a path in `always_allow` stays open to every crawler refused the whole site. With `refuse_all` selected every response an adapter finalizes carries `X-Robots-Tag: noindex, nofollow`. A module's answer is held in the key-value store until its refresh is due, a module that cannot answer has its last answer served, and with none held the answer is `503` and never an empty file. Without the section `/robots.txt` reaches the publisher's origin as before. - `ts keys rotate` and `ts keys deactivate` rotate and retire the request signing keys from an operator's machine. They write the two stores the service links as `jwks_store` and `signing_keys` through the `fastly` CLI, with the token from `FASTLY_API_TOKEN`. A private key reaches `fastly` on its standard input and never in its arguments, and a key id no platform can store is refused before anything is written. A command that cannot read the config store writes nothing, and a delete that failed part way can be run again. diff --git a/crates/trusted-server-core/src/middleware.rs b/crates/trusted-server-core/src/middleware.rs index baf97ae899..2a9e7c7efe 100644 --- a/crates/trusted-server-core/src/middleware.rs +++ b/crates/trusted-server-core/src/middleware.rs @@ -47,16 +47,12 @@ pub const HTML_MEDIA_TYPE: &str = "text/html"; /// bare `type/subtype` in lower case. /// /// HTML and the text assets that stream through core as text: every `text/` -/// type except `text/x-component`, whose rows are length prefixed, and -/// JavaScript, JSON and SVG. Anything else passes through untouched. +/// type except `text/x-component`, and JavaScript and JSON. #[must_use] pub fn is_dispatched_media_type(media_type: &str) -> bool { media_type != "text/x-component" && (media_type.starts_with("text/") - || matches!( - media_type, - "application/javascript" | "application/json" | "image/svg+xml" - )) + || matches!(media_type, "application/javascript" | "application/json")) } /// The type and subtype of a content type header, in lower case and without @@ -72,6 +68,20 @@ pub fn media_type_of(content_type: &str) -> String { .to_ascii_lowercase() } +/// Whether a content type declares no charset or declares UTF-8, the one a +/// middleware's stream processor is handed. A body in another charset passes +/// through untouched, because a processor that writes UTF-8 into it would +/// corrupt it. +#[must_use] +pub fn is_utf8_or_undeclared_charset(content_type: &str) -> bool { + content_type.split(';').skip(1).all(|parameter| { + parameter.split_once('=').is_none_or(|(name, value)| { + !name.trim().eq_ignore_ascii_case("charset") + || value.trim().trim_matches('"').eq_ignore_ascii_case("utf-8") + }) + }) +} + /// When a middleware runs, relative to the store of shared templates. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum MiddlewarePhase { @@ -338,11 +348,7 @@ pub trait Middleware: Send + Sync { fn middleware_id(&self) -> &'static str; /// Whether this middleware works on a response of `media_type`, given as - /// a bare `type/subtype`. HTML alone unless a middleware says otherwise, - /// and an entry naming it under a media type it does not handle is - /// refused when the registry is built. On a media type other than HTML - /// an action's stream processor alone runs, because head markup and the - /// element and text handlers are decisions about HTML. + /// a bare `type/subtype`. fn handles_media_type(&self, media_type: &str) -> bool { media_type == HTML_MEDIA_TYPE } @@ -445,14 +451,20 @@ impl PhaseEntries { for (index, entry) in self.0.iter().enumerate() { let at = format!("[[{phase}]] entry {}", index + 1); if entry.media_type != entry.media_type.to_ascii_lowercase() - || entry.media_type.contains([';', ' ', '*']) + || entry + .media_type + .contains(|c: char| c == ';' || c == '*' || c.is_whitespace()) + || entry + .media_type + .split_once('/') + .is_none_or(|(_, subtype)| subtype.is_empty()) || !is_dispatched_media_type(&entry.media_type) { return Err(format!( "{at} covers `{}`, which is not a media type a middleware runs on. Write a \ bare `type/subtype` in lower case with no parameters, being `text/html`, \ another `text/` type other than `text/x-component`, \ - `application/javascript`, `application/json` or `image/svg+xml`", + `application/javascript` or `application/json`", entry.media_type )); } diff --git a/crates/trusted-server-core/src/middleware/tests.rs b/crates/trusted-server-core/src/middleware/tests.rs index 2f33d32e68..2ec0ed4c0a 100644 --- a/crates/trusted-server-core/src/middleware/tests.rs +++ b/crates/trusted-server-core/src/middleware/tests.rs @@ -321,6 +321,11 @@ fn an_entry_for_a_media_type_a_middleware_does_not_run_on_is_refused() { "image/png", "application/octet-stream", "text/x-component", + "image/svg+xml", + "text/", + "text", + "text/plain\t", + "text/ plain", ] { let mut other = entry(None, &["example.a"]); other.media_type = media_type.to_owned(); @@ -344,7 +349,6 @@ fn an_entry_may_cover_a_text_media_type_that_is_not_html() { "text/css", "application/javascript", "application/json", - "image/svg+xml", ] { let mut text = entry(None, &["example.a"]); text.media_type = media_type.to_owned(); @@ -362,7 +366,7 @@ fn a_media_type_is_the_type_and_subtype_of_a_content_type_in_lower_case() { assert_eq!(media_type_of("application/json"), "application/json"); assert_eq!(media_type_of(""), ""); assert!(is_dispatched_media_type("text/plain")); - assert!(is_dispatched_media_type("image/svg+xml")); + assert!(!is_dispatched_media_type("image/svg+xml")); assert!( !is_dispatched_media_type("text/x-component"), "a length prefixed stream is not changed" @@ -370,6 +374,20 @@ fn a_media_type_is_the_type_and_subtype_of_a_content_type_in_lower_case() { assert!(!is_dispatched_media_type("image/png")); } +#[test] +fn a_body_is_handed_to_a_middleware_only_in_utf8_or_an_undeclared_charset() { + assert!(is_utf8_or_undeclared_charset("text/plain")); + assert!(is_utf8_or_undeclared_charset("text/plain; charset=utf-8")); + assert!(is_utf8_or_undeclared_charset( + "text/plain; Charset=\"UTF-8\"" + )); + assert!(is_utf8_or_undeclared_charset("text/plain; format=flowed")); + assert!(!is_utf8_or_undeclared_charset("text/plain; charset=utf-16")); + assert!(!is_utf8_or_undeclared_charset( + "text/plain; charset=iso-8859-1" + )); +} + #[test] fn a_chain_on_another_media_type_runs_a_stream_processor_alone() { let document_state = IntegrationDocumentState::default(); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 67c7618d64..175fd39f6b 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -657,9 +657,9 @@ struct PublisherBodyProcessor { /// A response that is not HTML, as the middleware the entries covering it /// are told of it. -struct TextDocument<'a> { - /// The type and subtype of the response, in lower case. - media_type: String, +struct TextResponse<'a> { + /// The response's `Content-Type`, parameters and all. + content_type: &'a str, request_path: &'a str, request_host: &'a str, request_scheme: &'a str, @@ -669,50 +669,53 @@ struct TextDocument<'a> { } /// The stream processors the entries covering a response that is not HTML -/// run on its body, phase by phase in the order given, or none when its -/// media type is not one a middleware runs on or no entry covers it. -/// -/// A fetch middleware is told nothing a request left, as on a page, and a -/// serve middleware is told what this reader's request left. +/// run on its body, the fetch entry's and then the serve entry's, or none +/// when its media type is not one a middleware runs on, its charset is not +/// UTF-8 or no entry covers it. Both phases run here because the template +/// cache stores HTML alone. /// /// # Errors /// -/// When a middleware decides about HTML on a document that is not HTML, or +/// When a middleware decides about HTML on a response that is not HTML, or /// asks for a selector that does not parse. fn middleware_stream_processors( - document: &TextDocument<'_>, - phases: &[MiddlewarePhase], + response: &TextResponse<'_>, settings: &Settings, integration_registry: &IntegrationRegistry, ) -> Result>, Report> { - use crate::middleware::{MiddlewareContext, is_dispatched_media_type}; + use crate::middleware::{ + MiddlewareContext, is_dispatched_media_type, is_utf8_or_undeclared_charset, + }; - if !is_dispatched_media_type(&document.media_type) { + let media_type = media_type_of(response.content_type); + if !is_dispatched_media_type(&media_type) + || !is_utf8_or_undeclared_charset(response.content_type) + { return Ok(Vec::new()); } let mut processors = Vec::new(); - for phase in phases { + for phase in &MiddlewarePhase::ALL { let chain = integration_registry.middleware_chain( settings.phase_entries(*phase), *phase, - &document.media_type, - document.request_path, + &media_type, + response.request_path, ); if chain.is_empty() { continue; } let document_state = IntegrationDocumentState::default(); if *phase == MiddlewarePhase::Serve { - document.request_state.seed(&document_state); + response.request_state.seed(&document_state); } let plan = chain .plan(&MiddlewareContext { phase: *phase, - request_host: document.request_host, - request_scheme: document.request_scheme, - origin_host: document.origin_host, + request_host: response.request_host, + request_scheme: response.request_scheme, + origin_host: response.origin_host, document_state: &document_state, - max_buffered_script_bytes: document.max_buffered_script_bytes, + max_buffered_script_bytes: response.max_buffered_script_bytes, }) .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; processors.extend(plan.processors); @@ -791,17 +794,9 @@ impl PublisherBodyProcessor { ¶ms.request_scheme, )) } else { - // The serve entry runs on each reader's copy of a stored response - // later, see `reader_serve_processor`, and here on one that is not - // stored. - let phases: &[MiddlewarePhase] = if params.template_cache_key.is_some() { - &[MiddlewarePhase::Fetch] - } else { - &MiddlewarePhase::ALL - }; let processors = middleware_stream_processors( - &TextDocument { - media_type: media_type_of(¶ms.content_type), + &TextResponse { + content_type: ¶ms.content_type, request_path: ¶ms.request_path, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, @@ -809,7 +804,6 @@ impl PublisherBodyProcessor { request_state: ¶ms.request_state, max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, }, - phases, settings, integration_registry, )?; @@ -907,14 +901,9 @@ fn process_response_streaming( .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; } else { - let phases: &[MiddlewarePhase] = if params.shared_template_authorized { - &[MiddlewarePhase::Fetch] - } else { - &MiddlewarePhase::ALL - }; let processors = middleware_stream_processors( - &TextDocument { - media_type: media_type_of(params.content_type), + &TextResponse { + content_type: params.content_type, request_path: params.request_path, request_host: params.request_host, request_scheme: params.request_scheme, @@ -922,7 +911,6 @@ fn process_response_streaming( request_state: params.request_state, max_buffered_script_bytes: params.settings.publisher.max_buffered_body_bytes, }, - phases, params.settings, params.integration_registry, )?; @@ -1710,29 +1698,7 @@ fn reader_serve_processor( use crate::middleware::{HTML_MEDIA_TYPE, MiddlewarePhase}; if !is_html_content_type(¶ms.content_type) { - let origin_host = settings.publisher.origin_host(); - let mut processors = middleware_stream_processors( - &TextDocument { - media_type: media_type_of(¶ms.content_type), - request_path: ¶ms.request_path, - request_host: ¶ms.request_host, - request_scheme: ¶ms.request_scheme, - origin_host: &origin_host, - request_state: ¶ms.request_state, - max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, - }, - &[MiddlewarePhase::Serve], - settings, - integration_registry, - )?; - return Ok(match processors.len() { - 0 => None, - 1 => processors.pop(), - _ => Some(Box::new(WithStreamProcessors { - inner: processors.remove(0), - processors, - })), - }); + return Ok(None); } let chain = integration_registry.middleware_chain( &settings.serve, @@ -20794,8 +20760,13 @@ mod tests { settings } - /// A text body streamed through the publisher path, as a reader gets it. - fn text_body_through_the_publisher(settings: &Settings, body: &str) -> String { + /// A text body of `content_type` streamed through the publisher path, as + /// a reader gets it. + fn text_body_through_the_publisher( + settings: &Settings, + content_type: &str, + body: &str, + ) -> String { let registry = IntegrationRegistry::new(settings).expect("should build a registry"); let params = OwnedProcessResponseParams { csp_nonce_observed: None, @@ -20808,7 +20779,7 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), request_path: "/a.txt".to_owned(), - content_type: "text/plain; charset=utf-8".to_string(), + content_type: content_type.to_string(), permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -20841,7 +20812,7 @@ mod tests { }]); let body = "User-agent: *\nSee https://origin.example.com/more\n"; - let output = text_body_through_the_publisher(&settings, body); + let output = text_body_through_the_publisher(&settings, "text/plain; charset=utf-8", body); assert_eq!( output, @@ -20929,7 +20900,7 @@ mod tests { middleware: vec![fixture::TEXT.to_owned()], }]); - let output = text_body_through_the_publisher(&settings, "plain\n"); + let output = text_body_through_the_publisher(&settings, "text/plain", "plain\n"); assert_eq!( output, "plain\n", @@ -20937,6 +20908,25 @@ mod tests { ); } + #[test] + fn a_text_body_in_a_charset_other_than_utf8_is_left_as_it_was() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT.to_owned()], + }]); + + let output = + text_body_through_the_publisher(&settings, "text/plain; charset=utf-16", "plain\n"); + + assert_eq!( + output, "plain\n", + "should hand a middleware no body it would corrupt by writing UTF-8" + ); + } + #[test] fn tsjs_dynamic_serves_a_standalone_module_by_registration_flag_not_by_name() { // The module the registration marks standalone is one a vendor crate diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 267c87aec2..8cf0c04882 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1737,23 +1737,23 @@ table. An entry carries none and switches nothing on. ### Entries for a media type other than HTML An entry covers `text/html`, another `text/` type other than -`text/x-component`, `application/javascript`, `application/json` or -`image/svg+xml`, written as a bare `type/subtype` in lower case with no -parameters. A response is matched by the type and subtype of its -`Content-Type`, without the parameters, so `text/plain; charset=utf-8` -takes the entry for `text/plain`. Anything else, an image or a length -prefixed stream among them, passes through untouched and no entry can -cover it. +`text/x-component`, `application/javascript` or `application/json`, written +as a bare `type/subtype` in lower case with no parameters. A response is +matched by the type and subtype of its `Content-Type`, without the +parameters, so `text/plain; charset=utf-8` takes the entry for +`text/plain`. Anything else, an image or a length prefixed stream among +them, passes through untouched and no entry can cover it. So does a +response that declares a charset other than UTF-8. A middleware says which media types it works on, HTML alone unless it says otherwise, and an entry naming one under a media type it does not handle is -refused when the settings load, with the entry and the middleware named. On -a media type other than HTML a middleware changes the body as a stream, +refused when the registry is built, with the entry and the middleware named. +On a media type other than HTML a middleware changes the body as a stream, after core has moved the origin's addresses in it, in the order the entry names them. Head markup and the element and text handlers are decisions about HTML, and a middleware that makes one there is refused by name. The -fetch entry's middleware run once on what is stored, and the serve entry's -on each reader's copy, as on a page. +template cache stores HTML alone, so the fetch entry's middleware and then +the serve entry's run on each response as it is fetched. ```toml [[fetch]] @@ -1787,7 +1787,7 @@ the names in and the order `trusted-server.example.toml` shows them in. An entry that could not do what it says refuses the configuration, both when a deployment is validated and when the settings load. That is a media type -other than `text/html`, a path that does not start with `/` or that holds a +no middleware runs on, a path that does not start with `/` or that holds a `*`, a `?` or a `#`, an entry that names no middleware or names one twice, a key an entry does not read, and an entry that an earlier one already covers. An entry naming a middleware that no running module supplies refuses startup, From 6b5b4866251bf4f0e6702fb3f527c586421fe6d7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 13 Oct 2026 12:38:35 +0100 Subject: [PATCH 4/8] A serve middleware's headers reach a response that is not HTML The entries covering a text response ran their middleware's stream processors and dropped the headers a serve middleware wrote. The headers now come back with the processors and are written on the response on both paths, as they are for a page. --- .../src/integrations/registry.rs | 45 +++++- .../src/middleware/tests.rs | 24 +++ crates/trusted-server-core/src/publisher.rs | 149 +++++++++++++++++- 3 files changed, 207 insertions(+), 11 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index b1e6e7ea01..63f38b8d21 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -2957,6 +2957,12 @@ pub(crate) mod test_support { pub(crate) const TEXT_MEDIA_TYPE: &str = "text/plain"; /// The line the text middleware adds at the end of a text body. pub(crate) const TEXT_LINE: &str = "# added by the middleware fixture\n"; + /// The name of the serve middleware that writes a header on a text + /// response. + pub(crate) const TEXT_HEADER: &str = "testing.middleware-fixture.text-header"; + /// The header the text header middleware writes, name and value. + pub(crate) const TEXT_HEADER_NAME: &str = "x-fixture-text"; + pub(crate) const TEXT_HEADER_VALUE: &str = "served"; /// The builder core's test build lists beside its own. pub(crate) const BUILDER: IntegrationBuilder = @@ -3118,6 +3124,32 @@ pub(crate) mod test_support { } } + struct TextHeader; + + impl Middleware for TextHeader { + fn middleware_id(&self) -> &'static str { + TEXT_HEADER + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == TEXT_MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + response_headers: vec![( + http::HeaderName::from_static(TEXT_HEADER_NAME), + http::HeaderValue::from_static(TEXT_HEADER_VALUE), + )], + ..MiddlewareAction::pass() + } + } + } + struct Broken; impl Middleware for Broken { @@ -3158,6 +3190,7 @@ pub(crate) mod test_support { .with_middleware(Arc::new(Reader)) .with_middleware(Arc::new(BrokenReader)) .with_middleware(Arc::new(Text)) + .with_middleware(Arc::new(TextHeader)) .build(), )) } @@ -6064,6 +6097,7 @@ mod tests { fixture::READER, fixture::BROKEN_READER, fixture::TEXT, + fixture::TEXT_HEADER, crate::service_version::MODULE, ], "should list the middleware the selected module supplies, and core's own after them" @@ -6083,6 +6117,7 @@ mod tests { [ fixture::READER, fixture::BROKEN_READER, + fixture::TEXT_HEADER, crate::service_version::MODULE ], "should list the middleware that run in the serve phase, core's own last" @@ -6199,13 +6234,14 @@ mod tests { message.contains( "[[fetch]] entry 2 names `testing.nothing`, which no module that runs supplies" ) && message.contains(&format!( - "[{}, {}, {}, {}, {}, {}, {}]", + "[{}, {}, {}, {}, {}, {}, {}, {}]", fixture::HEAD, fixture::LINKS, fixture::BROKEN, fixture::READER, fixture::BROKEN_READER, fixture::TEXT, + fixture::TEXT_HEADER, crate::service_version::MODULE )), "should name the entry and list what could be named: {message}" @@ -6259,6 +6295,7 @@ mod tests { fixture::READER, fixture::BROKEN_READER, fixture::TEXT, + fixture::TEXT_HEADER, ], "should list every middleware when there are no entries" ); @@ -6271,7 +6308,8 @@ mod tests { fixture::BROKEN, fixture::READER, fixture::BROKEN_READER, - fixture::TEXT + fixture::TEXT, + fixture::TEXT_HEADER ], "should leave out a middleware any entry names" ); @@ -6339,7 +6377,8 @@ mod tests { fixture::LINKS, fixture::BROKEN, fixture::BROKEN_READER, - fixture::TEXT + fixture::TEXT, + fixture::TEXT_HEADER ], "should count a middleware as named only by an entry of a phase it runs in" ); diff --git a/crates/trusted-server-core/src/middleware/tests.rs b/crates/trusted-server-core/src/middleware/tests.rs index d6055b9ac9..78b0329ae6 100644 --- a/crates/trusted-server-core/src/middleware/tests.rs +++ b/crates/trusted-server-core/src/middleware/tests.rs @@ -431,6 +431,30 @@ fn a_chain_on_another_media_type_runs_a_stream_processor_alone() { assert!(plan.head_inserts.is_empty()); } +#[test] +fn a_serve_chain_on_another_media_type_may_write_a_header() { + let document_state = IntegrationDocumentState::default(); + let context = super::test_support::context(MiddlewarePhase::Serve, &document_state); + + let plan = MiddlewareChain::new( + MiddlewarePhase::Serve, + "text/plain", + vec![header("example.version", "x-example", "1")], + ) + .plan(&context) + .expect("a header is not a decision about HTML"); + + assert_eq!( + plan.response_headers.len(), + 1, + "should keep the header for the response" + ); + assert!( + plan.processors.is_empty() && plan.head_inserts.is_empty(), + "should change nothing in the body" + ); +} + /// A stream processor that adds `text` at the end of a body. struct Append(String); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d713a5633b..cd82e58df7 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -673,7 +673,8 @@ struct TextResponse<'a> { } /// The stream processors the entries covering a response that is not HTML -/// run on its body, the fetch entry's and then the serve entry's, or none +/// run on its body, the fetch entry's and then the serve entry's, with the +/// headers the serve entry's middleware write on the response, or none /// when its media type is not one a middleware runs on, its charset is not /// UTF-8 or no entry covers it. Both phases run here because the template /// cache stores HTML alone. @@ -686,7 +687,7 @@ fn middleware_stream_processors( response: &TextResponse<'_>, settings: &Settings, integration_registry: &IntegrationRegistry, -) -> Result>, Report> { +) -> Result> { use crate::middleware::{ MiddlewareContext, is_dispatched_media_type, is_utf8_or_undeclared_charset, }; @@ -695,9 +696,10 @@ fn middleware_stream_processors( if !is_dispatched_media_type(&media_type) || !is_utf8_or_undeclared_charset(response.content_type) { - return Ok(Vec::new()); + return Ok(TextPass::default()); } let mut processors = Vec::new(); + let mut response_headers = Vec::new(); for phase in &MiddlewarePhase::ALL { let chain = integration_registry.middleware_chain( settings.phase_entries(*phase), @@ -724,8 +726,20 @@ fn middleware_stream_processors( }) .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; processors.extend(plan.processors); + response_headers.extend(plan.response_headers); } - Ok(processors) + Ok(TextPass { + processors, + response_headers, + }) +} + +/// What the entries covering a text response do to it: the processors over +/// its body and the headers the serve entry's middleware write. +#[derive(Default)] +struct TextPass { + processors: Vec>, + response_headers: Vec, } /// A processor followed by the ones the entries add, each handed what the @@ -802,7 +816,7 @@ impl PublisherBodyProcessor { ¶ms.request_scheme, )) } else { - let processors = middleware_stream_processors( + let pass = middleware_stream_processors( &TextResponse { content_type: ¶ms.content_type, request_path: ¶ms.request_path, @@ -815,6 +829,7 @@ impl PublisherBodyProcessor { settings, integration_registry, )?; + response_headers = pass.response_headers; Box::new(WithStreamProcessors { inner: Box::new(create_url_replacer( ¶ms.origin_host, @@ -822,7 +837,7 @@ impl PublisherBodyProcessor { ¶ms.request_host, ¶ms.request_scheme, )), - processors, + processors: pass.processors, }) }; @@ -924,7 +939,7 @@ fn process_response_streaming( .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; } else { - let processors = middleware_stream_processors( + let pass = middleware_stream_processors( &TextResponse { content_type: params.content_type, request_path: params.request_path, @@ -937,6 +952,7 @@ fn process_response_streaming( params.settings, params.integration_registry, )?; + response_headers = pass.response_headers; let processor = WithStreamProcessors { inner: Box::new(create_url_replacer( params.origin_host, @@ -944,7 +960,7 @@ fn process_response_streaming( params.request_host, params.request_scheme, )), - processors, + processors: pass.processors, }; StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) @@ -21174,6 +21190,123 @@ mod tests { ); } + /// The parameters of a text response at `/a.txt`, from the origin as + /// `content_type` says, for a reader of `proxy.example.com`. + fn text_params(content_type: &str) -> OwnedProcessResponseParams { + OwnedProcessResponseParams { + csp_nonce_observed: None, + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/a.txt".to_owned(), + content_type: content_type.to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + } + } + + /// Settings running the fixture with a serve entry on `text/plain` + /// naming its header-writing middleware. + fn settings_serving_the_text_header() -> Settings { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let mut settings = settings_running_the_middleware_fixture(Vec::new()); + settings.serve = PhaseEntries::new(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT_HEADER.to_owned()], + }]); + settings + } + + fn named_headers(headers: &[ResponseHeader]) -> Vec<(&str, &[u8])> { + headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())) + .collect() + } + + #[test] + fn a_text_body_s_serve_middleware_write_their_headers_on_the_response() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let settings = settings_serving_the_text_header(); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let params = text_params("text/plain; charset=utf-8"); + let mut output = Vec::new(); + + let headers = stream_publisher_body( + EdgeBody::from(b"plain\n".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should stream a text body"); + + assert_eq!( + named_headers(&headers), + [( + fixture::TEXT_HEADER_NAME, + fixture::TEXT_HEADER_VALUE.as_bytes() + )], + "should hand back the header the serve middleware writes on a text response" + ); + assert_eq!(String::from_utf8(output).expect("text"), "plain\n"); + } + + #[tokio::test] + async fn a_streamed_text_body_s_serve_middleware_write_their_headers_on_the_response() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::platform::test_support::noop_services; + + let settings = settings_serving_the_text_header(); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + let services = noop_services(); + let mut params = text_params("text/plain; charset=utf-8"); + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"pla"), + bytes::Bytes::from_static(b"in\n"), + ])); + let mut output = Vec::new(); + + let headers = stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("should stream a text body"); + + assert_eq!( + named_headers(&headers), + [( + fixture::TEXT_HEADER_NAME, + fixture::TEXT_HEADER_VALUE.as_bytes() + )], + "should hand back the header the serve middleware writes on a streamed text response" + ); + assert_eq!(String::from_utf8(output).expect("text"), "plain\n"); + } + #[test] fn a_text_body_no_entry_covers_is_left_as_it_was() { use crate::integrations::registry_test_support::middleware_fixture as fixture; From 443d1f402df7689a7e7f770a13bb079f51016191 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 08:38:35 +0100 Subject: [PATCH 5/8] A text response a serve middleware is handed request state for is private A page for a request that modules left state on is sent `private, no-store` and loses its validators, so no shared cache replays it to another reader. A text response a `[[serve]]` entry covers is handed the same state, but it kept the origin's caching directives, so a shared cache could replay a body shaped for one reader to the next. Both finalizers now apply the same policy to such a response, once the serve middleware's own headers are written, so none of them can undo it. `text_entry_media_type` holds the rule for which responses the entries run on, so the finalizers and the body processor decide alike. The test sends a request a module left state on for a text file through the publisher path and both finalizers, and checks that a request with no state keeps the origin's `Cache-Control`. --- .../src/integrations/registry.rs | 10 +- crates/trusted-server-core/src/publisher.rs | 178 +++++++++++++++++- 2 files changed, 176 insertions(+), 12 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index dfab00bd09..46efe12439 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -176,15 +176,17 @@ impl IntegrationDocumentState { /// /// A request preparer or a request filter leaves a value under its /// integration id with [`IntegrationRequestState::insert`]. Where the -/// request's reader is served an HTML document, every value is copied into -/// the [`IntegrationDocumentState`] the `[[serve]]` entry's middleware share, +/// request's reader is served an HTML document, or another body a +/// `[[serve]]` entry covers, every value is copied into the +/// [`IntegrationDocumentState`] the `[[serve]]` entry's middleware share, /// so the module's serve middleware reads it there, and the same values are /// handed to the module's response finalizer. /// /// A request that carries any value keeps to the origin path. Its document is /// never read from a shared template and never stored as one, and an HTML -/// response with a body is sent `private, no-store`. A fetch middleware is -/// handed none of these values. +/// response with a body, or another response a `[[serve]]` entry covers, is +/// sent `private, no-store`. A fetch middleware is handed none of these +/// values. #[derive(Clone, Default)] pub struct IntegrationRequestState { values: IntegrationDocumentStateMap, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index e50ddd7e5b..1afdcbd417 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -692,16 +692,11 @@ fn middleware_stream_processors( settings: &Settings, integration_registry: &IntegrationRegistry, ) -> Result> { - use crate::middleware::{ - MiddlewareContext, is_dispatched_media_type, is_utf8_or_undeclared_charset, - }; + use crate::middleware::MiddlewareContext; - let media_type = media_type_of(response.content_type); - if !is_dispatched_media_type(&media_type) - || !is_utf8_or_undeclared_charset(response.content_type) - { + let Some(media_type) = text_entry_media_type(response.content_type) else { return Ok(TextPass::default()); - } + }; let mut processors = Vec::new(); let mut response_headers = Vec::new(); for phase in &MiddlewarePhase::ALL { @@ -738,6 +733,48 @@ fn middleware_stream_processors( }) } +/// The media type the entries match a response that is not HTML by, or +/// `None` when no middleware runs on it: HTML and the React Server +/// Components stream, which have processors of their own, a type no entry +/// may cover, and a charset other than UTF-8. +fn text_entry_media_type(content_type: &str) -> Option { + use crate::middleware::{is_dispatched_media_type, is_utf8_or_undeclared_charset}; + + let media_type = media_type_of(content_type); + let runs = !is_html_content_type(content_type) + && !content_type_contains_ascii_case_insensitive(content_type, "text/x-component") + && is_dispatched_media_type(&media_type) + && is_utf8_or_undeclared_charset(content_type); + runs.then_some(media_type) +} + +/// Keeps a response that is not HTML out of shared caches when a `[[serve]]` +/// entry covers it and a module left state on the request, because the +/// entry's middleware are handed that state. Called once the serve +/// middleware's headers are written, so none of them can undo it. +fn apply_text_entry_cache_rules( + response: &mut Response, + params: &OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, +) { + let Some(media_type) = text_entry_media_type(¶ms.content_type) else { + return; + }; + let serve = MiddlewarePhase::Serve; + let served = !integration_registry + .middleware_chain( + settings.phase_entries(serve), + serve, + &media_type, + ¶ms.request_path, + ) + .is_empty(); + if served && !params.request_state.is_empty() { + enforce_terminal_private_cache_privacy(response); + } +} + /// What the entries covering a text response do to it: the processors over /// its body and the headers the serve entry's middleware write. #[derive(Default)] @@ -2112,6 +2149,7 @@ pub async fn buffer_publisher_response_async( ) .await?; write_response_headers(&mut response, response_headers); + apply_text_entry_cache_rules(&mut response, ¶ms, settings, integration_registry); // Authorized template cache transforms are emitted as identity by // `process_response_streaming_async`; inline transforms retain the origin // coding. This avoids recompressing and immediately decoding a full document. @@ -2928,6 +2966,7 @@ pub async fn publisher_response_into_streaming_response( } }; write_response_headers(&mut response, processor.take_response_headers()); + apply_text_entry_cache_rules(&mut response, ¶ms, &settings, integration_registry); // The guard is created before the lazy stream so an auction whose // response body is dropped unpolled still logs the loss. let dispatched_auction = params.dispatched_auction.take().map(|dispatched| { @@ -21382,6 +21421,129 @@ mod tests { assert_eq!(String::from_utf8(output).expect("text"), "plain\n"); } + /// The answers to `GET /a.txt` through the publisher path, the origin + /// answering `status` with `plain\n` as UTF-8 text and the headers given, + /// finalized as the adapters that buffer do and then as Fastly does. + /// With `marked`, a module leaves state on the request. + async fn text_answers_through_the_publisher( + settings: Settings, + marked: bool, + status: u16, + origin_headers: &[(&str, &str)], + ) -> Vec<(Response, String)> { + let settings = Arc::new(settings); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + let mut answers = Vec::new(); + for streaming in [false, true] { + let stub = Arc::new(StubHttpClient::new()); + let mut headers = vec![("content-type", "text/plain; charset=utf-8")]; + headers.extend_from_slice(origin_headers); + stub.push_response_with_headers(status, b"plain\n".to_vec(), headers); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/a.txt") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + if marked { + crate::integrations::registry_test_support::request_fixture::mark(&mut request); + } + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test_with_ip(None, consent, None); + let proxied = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy the request"); + let response = if streaming { + publisher_response_into_streaming_response( + proxied, + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await + } else { + buffer_publisher_response_async( + proxied, + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + } + .expect("should finalize the response"); + let (parts, body) = response.into_parts(); + let body = match body { + EdgeBody::Stream(mut stream) => { + let mut out = Vec::new(); + while let Some(chunk) = futures::StreamExt::next(&mut stream).await { + out.extend_from_slice(&chunk.expect("should read a chunk")); + } + out + } + other => other.into_bytes().expect("should hold the bytes").to_vec(), + }; + answers.push(( + Response::from_parts(parts, EdgeBody::empty()), + String::from_utf8(body).expect("text"), + )); + } + answers + } + + #[tokio::test] + async fn a_text_response_whose_serve_entry_is_handed_request_state_is_private() { + let origin = [("cache-control", "public, max-age=600"), ("etag", "\"v1\"")]; + for (marked, cache_control) in [(true, "no-store, private"), (false, "public, max-age=600")] + { + let answers = text_answers_through_the_publisher( + settings_serving_the_text_header(), + marked, + 200, + &origin, + ) + .await; + for (response, _) in &answers { + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "should keep a text response shaped from one reader's state out of \ + shared caches, and no other" + ); + } + } + } + #[test] fn a_text_body_no_entry_covers_is_left_as_it_was() { use crate::integrations::registry_test_support::middleware_fixture as fixture; From c024839804d5d129c792238ae2bc2e3654ea88bc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 08:38:37 +0100 Subject: [PATCH 6/8] A middleware runs on a text body only when the status is 200 Every covered text response ran the entries' middleware whatever its status, so a line added to a `206` slice left its `Content-Range` untrue, and an error page was changed as if it were the file. Only a `200` carries the whole file, so the entries' middleware now run on that alone. Core still moves the origin's addresses in every text response, as before. The status travels with the response's other details in the parameters of the body processing. The test sends a `200`, a `206`, a `404` and a `500` text response through the publisher path and both finalizers. --- crates/trusted-server-core/src/publisher.rs | 83 +++++++++++++++++++-- 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1afdcbd417..9ed2b32bc1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -642,6 +642,8 @@ struct ProcessResponseParams<'a> { request_path: &'a str, settings: &'a Settings, content_type: &'a str, + /// See [`OwnedProcessResponseParams::status`]. + status: StatusCode, integration_registry: &'a IntegrationRegistry, /// Head script carrying this request's permission state, or [`None`] under a /// shared-template mode. See [`template_permissions_script`]. @@ -668,6 +670,7 @@ struct PublisherBodyProcessor { struct TextResponse<'a> { /// The response's `Content-Type`, parameters and all. content_type: &'a str, + status: StatusCode, request_path: &'a str, request_host: &'a str, request_scheme: &'a str, @@ -679,9 +682,8 @@ struct TextResponse<'a> { /// The stream processors the entries covering a response that is not HTML /// run on its body, the fetch entry's and then the serve entry's, with the /// headers the serve entry's middleware write on the response, or none -/// when its media type is not one a middleware runs on, its charset is not -/// UTF-8 or no entry covers it. Both phases run here because the template -/// cache stores HTML alone. +/// when [`text_entry_media_type`] gives no media type or no entry covers +/// it. Both phases run here because the template cache stores HTML alone. /// /// # Errors /// @@ -694,7 +696,7 @@ fn middleware_stream_processors( ) -> Result> { use crate::middleware::MiddlewareContext; - let Some(media_type) = text_entry_media_type(response.content_type) else { + let Some(media_type) = text_entry_media_type(response.content_type, response.status) else { return Ok(TextPass::default()); }; let mut processors = Vec::new(); @@ -736,12 +738,15 @@ fn middleware_stream_processors( /// The media type the entries match a response that is not HTML by, or /// `None` when no middleware runs on it: HTML and the React Server /// Components stream, which have processors of their own, a type no entry -/// may cover, and a charset other than UTF-8. -fn text_entry_media_type(content_type: &str) -> Option { +/// may cover, a charset other than UTF-8, and a status other than `200`. +/// Only a `200` carries the whole file, so a partial `206` or an error +/// body is left as the origin sent it. +fn text_entry_media_type(content_type: &str, status: StatusCode) -> Option { use crate::middleware::{is_dispatched_media_type, is_utf8_or_undeclared_charset}; let media_type = media_type_of(content_type); - let runs = !is_html_content_type(content_type) + let runs = status == StatusCode::OK + && !is_html_content_type(content_type) && !content_type_contains_ascii_case_insensitive(content_type, "text/x-component") && is_dispatched_media_type(&media_type) && is_utf8_or_undeclared_charset(content_type); @@ -758,7 +763,7 @@ fn apply_text_entry_cache_rules( settings: &Settings, integration_registry: &IntegrationRegistry, ) { - let Some(media_type) = text_entry_media_type(¶ms.content_type) else { + let Some(media_type) = text_entry_media_type(¶ms.content_type, params.status) else { return; }; let serve = MiddlewarePhase::Serve; @@ -860,6 +865,7 @@ impl PublisherBodyProcessor { let pass = middleware_stream_processors( &TextResponse { content_type: ¶ms.content_type, + status: params.status, request_path: ¶ms.request_path, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, @@ -983,6 +989,7 @@ fn process_response_streaming( let pass = middleware_stream_processors( &TextResponse { content_type: params.content_type, + status: params.status, request_path: params.request_path, request_host: params.request_host, request_scheme: params.request_scheme, @@ -2023,6 +2030,9 @@ pub struct OwnedProcessResponseParams { /// `[[serve]]` entries that cover the document. pub(crate) request_path: String, pub(crate) content_type: String, + /// The origin's status, which decides whether the entries' middleware + /// run on a body that is not HTML. See [`text_entry_media_type`]. + pub(crate) status: StatusCode, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: AdBidsState, /// Observation context for the in-flight auction. @@ -2504,6 +2514,7 @@ fn build_template_assembly_params( // Set by the caller, with the rest of what belongs to the request. request_path: String::new(), content_type: entry.metadata.content_type.clone(), + status: StatusCode::OK, // The template already carries the head seam; re-injecting would duplicate it. permissions_json, ad_slots_script: None, @@ -3299,6 +3310,7 @@ pub fn stream_publisher_body( request_path: ¶ms.request_path, settings, content_type: ¶ms.content_type, + status: params.status, integration_registry, permissions_script: permissions_script.as_deref(), ad_slots_script: params.ad_slots_script.as_deref(), @@ -5920,6 +5932,7 @@ pub async fn handle_publisher_request( request_scheme: request_scheme.to_string(), request_path: request_path.clone(), content_type, + status, permissions_json, ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), @@ -9564,6 +9577,7 @@ mod tests { request_scheme: "https".to_owned(), request_path: "/".to_owned(), content_type: "application/json".to_owned(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -21194,6 +21208,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/a.txt".to_owned(), content_type: content_type.to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -21267,6 +21282,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/a.txt".to_owned(), content_type: "text/plain; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -21319,6 +21335,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/a.txt".to_owned(), content_type: content_type.to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -21544,6 +21561,33 @@ mod tests { } } + #[tokio::test] + async fn a_text_body_is_handed_to_a_middleware_only_when_the_status_is_200() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT.to_owned()], + }]); + for (status, body) in [ + (200, format!("plain\n{}", fixture::TEXT_LINE)), + (206, "plain\n".to_owned()), + (404, "plain\n".to_owned()), + (500, "plain\n".to_owned()), + ] { + let answers = + text_answers_through_the_publisher(settings.clone(), false, status, &[]).await; + for (response, text) in &answers { + assert_eq!(response.status().as_u16(), status); + assert_eq!( + text, &body, + "should change a whole file and leave a {status} as the origin sent it" + ); + } + } + } + #[test] fn a_text_body_no_entry_covers_is_left_as_it_was() { use crate::integrations::registry_test_support::middleware_fixture as fixture; @@ -21768,6 +21812,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/css".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -21831,6 +21876,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -21915,6 +21961,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -22008,6 +22055,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -22116,6 +22164,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -22165,6 +22214,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -22294,6 +22344,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/css".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -22361,6 +22412,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/css".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -22431,6 +22483,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/css".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -22501,6 +22554,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/css".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -22571,6 +22625,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/css".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -22621,6 +22676,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/css".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -22847,6 +22903,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -22925,6 +22982,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -23005,6 +23063,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/css".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -23079,6 +23138,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/css".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -23235,6 +23295,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -23911,6 +23972,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -24109,6 +24171,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -24191,6 +24254,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "Text/HTML; Charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -24256,6 +24320,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -24376,6 +24441,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -24436,6 +24502,7 @@ mod tests { request_scheme: "https".to_string(), request_path: "/".to_owned(), content_type: "text/html".to_string(), + status: http::StatusCode::OK, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), From e0965ef78950696f52641adb1a78aa5290d74e56 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 08:38:39 +0100 Subject: [PATCH 7/8] A text body an entry covers loses the origin's validators A body a middleware changes kept the origin's `ETag` and `Last-Modified`, so a reader asking again with them got a `304` from the origin while the file was unchanged, and kept a body without what the middleware would add now. The headers go out before a streamed body is processed, so no new validator can be computed, and both finalizers remove the two headers from a response an entry covers. The test checks both finalizers, and that a body no entry covers keeps both headers. --- crates/trusted-server-core/src/publisher.rs | 59 +++++++++++++++++---- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 9ed2b32bc1..3be9a6371e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -753,8 +753,12 @@ fn text_entry_media_type(content_type: &str, status: StatusCode) -> Option Date: Fri, 16 Oct 2026 08:38:42 +0100 Subject: [PATCH 8/8] The guides say which text answers a middleware changes The changelog entry, the configuration guide and the integration guide now say that a middleware runs on a text body only for a `200`, that a response an entry covers loses the origin's validators, and that one a `[[serve]]` entry covers is sent `private, no-store` when a module left state on the request. --- CHANGELOG.md | 2 +- docs/guide/configuration.md | 8 ++++++-- docs/guide/integration-guide.md | 7 ++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71c3ab1d83..615c78759b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,7 +82,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- A `[[fetch]]` or `[[serve]]` entry may cover a media type other than `text/html`: another `text/` type except `text/x-component`, `application/javascript` or `application/json`, written as a bare `type/subtype` in lower case. A response that declares a charset other than UTF-8 passes through untouched. A middleware says which media types it works on with `handles_media_type`, HTML alone unless it says otherwise, and an entry naming one under a media type it does not handle is refused when the registry is built. On a media type other than HTML a middleware's stream processor alone runs on the body, after core has moved the origin's addresses, in the fetch phase and then the serve phase, and an action that decides about HTML there is refused by name. +- A `[[fetch]]` or `[[serve]]` entry may cover a media type other than `text/html`: another `text/` type except `text/x-component`, `application/javascript` or `application/json`, written as a bare `type/subtype` in lower case. A response that declares a charset other than UTF-8, or whose status is not `200`, gets no middleware. A middleware says which media types it works on with `handles_media_type`, HTML alone unless it says otherwise, and an entry naming one under a media type it does not handle is refused when the registry is built. On a media type other than HTML a middleware's stream processor alone runs on the body, after core has moved the origin's addresses, in the fetch phase and then the serve phase, and an action that decides about HTML there is refused by name. A response an entry covers loses the origin's `ETag` and `Last-Modified`, and one a `[[serve]]` entry covers is sent `private, no-store` when a module left state on the request. - The InMobi Choice consent prompt as a module, `[cmp] module = "inmobi"` with `[cmp.inmobi] script_url`, from `crates/cmp/inmobi`. Where a `[[fetch]]` entry names `cmp.inmobi`, its middleware writes the IAB TCF stub, InMobi's GPP stub and the deferred Choice loader into the head, in that order, and the two stubs' behavior is tested with Node's test runner, which CI now runs for every module's `tests/*.test.mjs`. The script URL is refused unless it is `https` on an InMobi Choice host with no user name or password, and a selected prompt that no entry places is refused. The Trusted Server maintainers hold the crate until InMobi adopt it. - `[robots-txt]` serves a publisher's `/robots.txt` from the settings. `modules` names what makes the file, in order. `refuse_all` and `allow_all` are Trusted Server's own, and any other name is a module that contributes rules through the new `RobotsTxtContributor` extension point, declared with `with_robots_txt_contributor`. Trusted Server writes the file, so a contributor cannot produce a malformed one, a refusal cannot be loosened, and a path in `always_allow` stays open to every crawler refused the whole site. With `refuse_all` selected every response an adapter finalizes carries `X-Robots-Tag: noindex, nofollow`. A module's answer is held in the key-value store until its refresh is due, a module that cannot answer has its last answer served, and with none held the answer is `503` and never an empty file. Without the section `/robots.txt` reaches the publisher's origin as before. - `ts keys rotate` and `ts keys deactivate` rotate and retire the request signing keys from an operator's machine. They write the two stores the service links as `jwks_store` and `signing_keys` through the `fastly` CLI, with the token from `FASTLY_API_TOKEN`. A private key reaches `fastly` on its standard input and never in its arguments, and a key id no platform can store is refused before anything is written. A command that cannot read the config store writes nothing, and a delete that failed part way can be run again. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index bbab79dbab..b23e538636 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1761,7 +1761,8 @@ matched by the type and subtype of its `Content-Type`, without the parameters, so `text/plain; charset=utf-8` takes the entry for `text/plain`. Anything else, an image or a length prefixed stream among them, passes through untouched and no entry can cover it. So does a -response that declares a charset other than UTF-8. +response that declares a charset other than UTF-8 or whose status is not +`200`, such as a partial `206` or an error page. A middleware says which media types it works on, HTML alone unless it says otherwise, and an entry naming one under a media type it does not handle is @@ -1771,7 +1772,10 @@ after core has moved the origin's addresses in it, in the order the entry names them. Head markup and the element and text handlers are decisions about HTML, and a middleware that makes one there is refused by name. The template cache stores HTML alone, so the fetch entry's middleware and then -the serve entry's run on each response as it is fetched. +the serve entry's run on each response as it is fetched. A response an entry +covers loses the origin's `ETag` and `Last-Modified`, because a middleware +may change its body, and when a module left state on the request, one a +`[[serve]]` entry covers is sent `private, no-store`, as a page is. ```toml [[fetch]] diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index 1eff894c8b..d808801231 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -204,9 +204,10 @@ A middleware works on `text/html` unless it says otherwise, with handle is refused when the registry is built. On a media type other than HTML an action may carry a stream processor alone, which is handed the body after core has moved the origin's addresses in it, chunk by chunk, with the -last chunk marked. Head markup and the element and text handlers are -decisions about HTML, and an action that makes one on another media type is -refused by name when the chain is planned. +last chunk marked, and only for a `200`, whose body is the whole file. Head +markup and the element and text handlers are decisions about HTML, and an +action that makes one on another media type is refused by name when the +chain is planned. ```rust struct Sellers;