From 49b280292bd97d13385f0531aa4de7de4ea584b0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 11 Oct 2026 11:23:40 +0100 Subject: [PATCH 01/23] robots.txt is a middleware named for its path Core's own middleware `robots_txt`, in `robots_txt/middleware.rs`, makes the file `[robots-txt]` describes where a `[[fetch]]` or `[[serve]]` entry for `text/plain` on `/robots.txt` names it. Its preparation does what the route's handler did, a refusal of every crawler from nothing and otherwise every selected contributor's held answer assembled, and answers a refusal with `503` and `Retry-After` when a contributor fails with nothing held, so nothing is stored and no empty file is served. Its action replaces whatever the origin sent with the file, and the creating rule makes the file where the origin has none. The registry makes the middleware itself once every registration is in, because the contributors it asks are known only then and a middleware cannot reach the registry. The settings refuse a `[robots-txt]` section that no entry places, because the file would never be made. The route goes from the four adapters, with `handle_robots_txt`, the response builder and the route's own cache lifetime. The file takes the deployment's cache policy for the path, as every file does. The adapters' route tests go with it, and each now proves the path is no route of this server; the file itself is proven through the publisher path in core. The stock list carries the middleware on `text/plain`, and the example configuration places it. --- crates/trusted-server-adapter-axum/src/app.rs | 29 +- .../tests/seam_probe.rs | 133 +------ .../src/app.rs | 18 - .../tests/routes.rs | 97 +---- .../trusted-server-adapter-fastly/src/app.rs | 27 -- .../src/app/seam_probe_tests.rs | 113 +----- crates/trusted-server-adapter-spin/src/app.rs | 27 -- .../tests/routes.rs | 97 +---- .../src/integrations/registry.rs | 15 +- .../trusted-server-core/src/module_context.rs | 12 + crates/trusted-server-core/src/publisher.rs | 122 ++++++- .../src/robots_txt/middleware.rs | 229 ++++++++++++ .../trusted-server-core/src/robots_txt/mod.rs | 332 ++++++++---------- crates/trusted-server-core/src/settings.rs | 46 ++- crates/trusted-server-modules/src/lib.rs | 71 +++- trusted-server.example.toml | 12 +- 16 files changed, 699 insertions(+), 681 deletions(-) create mode 100644 crates/trusted-server-core/src/robots_txt/middleware.rs diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index e58be73e78..99b8570e1a 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -39,7 +39,6 @@ use trusted_server_core::publisher::{ use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, }; -use trusted_server_core::robots_txt::{ROBOTS_TXT_PATH, handle_robots_txt}; use trusted_server_core::settings::Settings; use trusted_server_core::settings_data::{ default_config_key, default_config_store_name, get_settings_from_config_store_with, @@ -451,7 +450,6 @@ enum NamedRouteHandler { FirstPartyClick, FirstPartySign, FirstPartyProxyRebuild, - RobotsTxt, } struct NamedRoute { @@ -460,7 +458,7 @@ struct NamedRoute { handler: NamedRouteHandler, } -fn named_routes() -> [NamedRoute; 15] { +fn named_routes() -> [NamedRoute; 14] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -545,28 +543,9 @@ fn named_routes() -> [NamedRoute; 15] { primary_methods: &[Method::GET, Method::POST], handler: NamedRouteHandler::FirstPartyProxyRebuild, }, - // The publisher's robots.txt, served only when the settings carry a - // `[robots-txt]` section, see `named_route_is_served`. - NamedRoute { - path: ROBOTS_TXT_PATH, - primary_methods: &[Method::GET, Method::HEAD], - handler: NamedRouteHandler::RobotsTxt, - }, ] } -/// Whether a named route is served for these settings. -/// -/// `/robots.txt` is this server's only when the publisher's settings carry a -/// `[robots-txt]` section. Without one the path reaches the publisher's -/// origin like any other, so a publisher keeping their own file is left alone. -fn named_route_is_served(settings: &Settings, handler: NamedRouteHandler) -> bool { - match handler { - NamedRouteHandler::RobotsTxt => settings.robots_txt.is_some(), - _ => true, - } -} - fn named_route_handler( state: Arc, handler: NamedRouteHandler, @@ -641,9 +620,6 @@ fn named_route_handler( NamedRouteHandler::FirstPartyProxyRebuild => { handle_first_party_proxy_rebuild(&state.settings, &services, req).await } - NamedRouteHandler::RobotsTxt => { - handle_robots_txt(&state.settings, &services, &state.registry, req).await - } } }, )) @@ -823,9 +799,6 @@ fn build_router(state: &Arc) -> RouterService { }); for route in named_routes() { - if !named_route_is_served(&state.settings, route.handler) { - continue; - } for method in route.primary_methods { router = router.route( route.path, diff --git a/crates/trusted-server-adapter-axum/tests/seam_probe.rs b/crates/trusted-server-adapter-axum/tests/seam_probe.rs index bb815ed33f..02c9c1a113 100644 --- a/crates/trusted-server-adapter-axum/tests/seam_probe.rs +++ b/crates/trusted-server-adapter-axum/tests/seam_probe.rs @@ -256,97 +256,6 @@ async fn proxy_route_reports_the_modules_geo_and_that_the_preparer_ran() { ); } -/// `[robots-txt] modules` naming the probe serves the rules the probe gives, -/// asked for the request's host and path and never its query, because the -/// answer is held and served to every crawler. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn robots_txt_selection_naming_a_module_serves_the_rules_it_gives() { - let settings = settings_with(&format!( - r#" - [robots-txt] - modules = ["testing.seam-probe", "allow_all"] - sitemap = "https://test-publisher.example.com/sitemap.xml" - {PROBE_BLOCK} - "# - )); - let mut service = service_with(settings, &[seam_probe::builder()]); - - let response = get(&mut service, "/robots.txt?crawler=one").await; - - assert_eq!( - response.status().as_u16(), - 200, - "the file should be served here and not by the origin" - ); - assert_eq!( - response - .headers() - .get("content-type") - .and_then(|value| value.to_str().ok()), - Some("text/plain; charset=utf-8") - ); - let file = body_text(response).await; - let agent = seam_probe::SEAM_PROBE_ROBOTS_AGENT; - assert!( - file.starts_with(&format!("User-Agent: {agent}\nDisallow: /")), - "the probe's rules should come first, as `modules` orders them: {file}" - ); - assert!( - file.contains("/robots.txt?\n"), - "the probe should be asked for the path with no query: {file}" - ); - assert!( - !file.contains("crawler=one"), - "nothing one crawler put in the address should reach a held answer: {file}" - ); - assert!( - file.ends_with( - "User-Agent: *\nAllow: /\n\nSitemap: https://test-publisher.example.com/sitemap.xml\n" - ), - "core's own rules and the sitemap should follow: {file}" - ); -} - -/// With `refuse_all` selected the file is the refusal, and every response -/// this adapter finalizes says not to index, the response of another route -/// included. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn refusing_every_crawler_tags_every_response() { - let settings = settings_with(&format!( - r#" - [robots-txt] - modules = "refuse_all" - always_allow = ["/ads.txt"] - {PROBE_BLOCK} - "# - )); - let mut service = service_with(settings, &[seam_probe::builder()]); - let tag = |response: &axum::response::Response| { - response - .headers() - .get("x-robots-tag") - .and_then(|value| value.to_str().ok()) - .map(str::to_owned) - }; - - let file = get(&mut service, "/robots.txt").await; - - assert_eq!(file.status().as_u16(), 200, "the refusal should be served"); - assert_eq!(tag(&file).as_deref(), Some("noindex, nofollow")); - assert_eq!( - body_text(file).await, - "User-agent: *\nAllow: /ads.txt\nDisallow: /\n" - ); - - let other = get(&mut service, seam_probe::SEAM_PROBE_REPORT_PATH).await; - - assert_eq!( - tag(&other).as_deref(), - Some("noindex, nofollow"), - "no route may be the one page a crawler is allowed to index" - ); -} - /// A name `[robots-txt] modules` gives that nothing the deployment runs /// supplies stops the adapter at startup, so the file does not fail on the /// first crawler to ask for it. @@ -356,6 +265,11 @@ fn robots_txt_selection_naming_a_module_that_is_not_running_fails_at_startup() { r#" [robots-txt] modules = ["testing.seam-probe"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] "#, ); @@ -374,43 +288,6 @@ fn robots_txt_selection_naming_a_module_that_is_not_running_fails_at_startup() { ); } -/// The methods this adapter registers for `/robots.txt` under `settings`. -fn robots_txt_methods(settings: Settings) -> Vec { - TrustedServerApp::routes_with_registrations(settings, &[seam_probe::builder()]) - .expect("should build the routes") - .routes() - .into_iter() - .filter(|route| route.path() == "/robots.txt") - .map(|route| route.method().to_string()) - .collect() -} - -/// With no `[robots-txt]` section the path is the publisher's, and the probe -/// being on offer changes nothing. With the section the path is this -/// server's for `GET` and `HEAD`. -#[test] -fn robots_txt_is_this_servers_only_when_the_settings_carry_the_section() { - assert_eq!( - robots_txt_methods(settings_with(PROBE_BLOCK)), - Vec::::new(), - "without the section no route of this server names the path" - ); - - let methods = robots_txt_methods(settings_with(&format!( - r#" - [robots-txt] - modules = "refuse_all" - {PROBE_BLOCK} - "# - ))); - for method in ["GET", "HEAD"] { - assert!( - methods.iter().any(|registered| registered == method), - "{method} /robots.txt should be answered here: {methods:?}" - ); - } -} - /// A request prepares exactly once whether it is served by a named route or by /// the fallback, so the invariant holds across the whole route table rather /// than only on the path the probe's own proxy route sits on. diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 21924ddc68..0d56bb2eee 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -45,7 +45,6 @@ use trusted_server_core::publisher::{ use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, }; -use trusted_server_core::robots_txt::{ROBOTS_TXT_PATH, handle_robots_txt}; use trusted_server_core::settings::Settings; use crate::middleware::{FinalizeResponseMiddleware, SanitizeRequestMiddleware}; @@ -899,23 +898,6 @@ fn build_router(state: &Arc) -> RouterService { } } - // `/robots.txt` is this server's only when the settings carry a - // `[robots-txt]` section. Without one the path reaches the publisher's - // origin like any other, so a publisher keeping their own file is - // left alone. - if state.settings.robots_txt.is_some() { - let robots_txt = make_handler(Arc::clone(&state), |s, services, req| async move { - handle_robots_txt(&s.settings, &services, &s.registry, req).await - }); - for method in publisher_fallback_methods() { - if matches!(method, Method::GET | Method::HEAD) { - router = router.route(ROBOTS_TXT_PATH, method, robots_txt.clone()); - } else { - router = router.route(ROBOTS_TXT_PATH, method, fallback.clone()); - } - } - } - for method in publisher_fallback_methods() { router = router.route("/", method.clone(), fallback.clone()); router = router.route("/{*rest}", method, fallback.clone()); diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index fae32e6683..824c8e9fa8 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -597,6 +597,11 @@ fn router_refusing_every_crawler() -> RouterService { [robots-txt] modules = "refuse_all" always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] "#, ) .expect("should parse route test settings"); @@ -621,22 +626,19 @@ fn robots_tag(response: &Response) -> Option { .map(str::to_owned) } +/// The file itself is the publisher path's, made by the `robots_txt` +/// middleware the entry names, so no route of this server names the path. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn refusing_every_crawler_serves_the_refusal_and_tags_every_response() { - let file = route( - router_refusing_every_crawler(), - robots_request("GET", "/robots.txt"), - ) - .await; - - assert_eq!( - file.status().as_u16(), - 200, - "the refusal should be served here" +async fn refusing_every_crawler_tags_every_response_and_registers_no_route() { + let registered: Vec = router_refusing_every_crawler() + .routes() + .into_iter() + .map(|route| route.path().to_string()) + .collect(); + assert!( + !registered.iter().any(|path| path == "/robots.txt"), + "the file is the publisher path's: {registered:?}" ); - assert_eq!(robots_tag(&file).as_deref(), Some("noindex, nofollow")); - let body = file.into_body().into_bytes().unwrap_or_default(); - assert_eq!(&body[..], b"User-agent: *\nAllow: /ads.txt\nDisallow: /\n"); let other = route( router_refusing_every_crawler(), @@ -651,73 +653,6 @@ async fn refusing_every_crawler_serves_the_refusal_and_tags_every_response() { ); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_head_of_robots_txt_is_answered_here_with_no_body() { - let response = route( - router_refusing_every_crawler(), - robots_request("HEAD", "/robots.txt"), - ) - .await; - - assert_eq!( - response.status().as_u16(), - 200, - "a HEAD should be answered here as a GET is" - ); - assert_eq!( - response - .headers() - .get("content-type") - .and_then(|value| value.to_str().ok()), - Some("text/plain; charset=utf-8") - ); - let body = response.into_body().into_bytes().unwrap_or_default(); - assert!(body.is_empty(), "a HEAD carries no body"); -} - -/// A method that is neither `GET` nor `HEAD` is the publisher's, as it is on -/// any other path, so it is not answered with the file. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn another_method_on_robots_txt_is_not_answered_with_the_file() { - let response = route( - router_refusing_every_crawler(), - robots_request("POST", "/robots.txt"), - ) - .await; - - let body = response.into_body().into_bytes().unwrap_or_default(); - assert!( - !body.starts_with(b"User-agent:"), - "a POST should reach the publisher fallback and not be given the file" - ); -} - -#[test] -fn robots_txt_is_this_servers_only_when_the_settings_carry_the_section() { - let registered = |router: RouterService| -> Vec<(String, String)> { - router - .routes() - .into_iter() - .map(|route| (route.method().to_string(), route.path().to_string())) - .collect() - }; - - let without = registered(test_router()); - assert!( - !without.iter().any(|(_, path)| path == "/robots.txt"), - "without the section the path is the publisher's: {without:?}" - ); - - let with = registered(router_refusing_every_crawler()); - for method in ["GET", "HEAD"] { - assert!( - with.iter() - .any(|(registered, path)| registered == method && path == "/robots.txt"), - "{method} /robots.txt should be answered here: {with:?}" - ); - } -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_response_carries_no_blanket_rule_without_the_section() { let response = route( diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 26ecf76fe5..7aeb36035f 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -142,7 +142,6 @@ use trusted_server_core::publisher::{ use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, }; -use trusted_server_core::robots_txt::{ROBOTS_TXT_PATH, handle_robots_txt}; use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{ DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store_with, @@ -962,9 +961,6 @@ async fn run_named_route( NamedRouteHandler::FirstPartyProxyRebuild => { handle_first_party_proxy_rebuild(&state.settings, services, req).await } - NamedRouteHandler::RobotsTxt => { - handle_robots_txt(&state.settings, services, &state.registry, req).await - } } } @@ -1318,7 +1314,6 @@ enum NamedRouteHandler { FirstPartyClick, FirstPartySign, FirstPartyProxyRebuild, - RobotsTxt, } struct NamedRoute { @@ -1437,27 +1432,8 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::GET, Method::POST], handler: NamedRouteHandler::FirstPartyProxyRebuild, }, - // The publisher's robots.txt, served only when the settings carry a - // `[robots-txt]` section, see `named_route_is_served`. - NamedRoute { - path: ROBOTS_TXT_PATH, - primary_methods: &[Method::GET, Method::HEAD], - handler: NamedRouteHandler::RobotsTxt, - }, ]; -/// Whether a named route is served for these settings. -/// -/// `/robots.txt` is this server's only when the publisher's settings carry a -/// `[robots-txt]` section. Without one the path reaches the publisher's -/// origin like any other, so a publisher keeping their own file is left alone. -fn named_route_is_served(settings: &Settings, handler: NamedRouteHandler) -> bool { - match handler { - NamedRouteHandler::RobotsTxt => settings.robots_txt.is_some(), - _ => true, - } -} - fn named_route_handler( state: Arc, handler: NamedRouteHandler, @@ -1560,9 +1536,6 @@ impl TrustedServerApp { // non-primary publisher fallback method is registered from the same // row. Adding a named route now requires editing only this table. for route in NAMED_ROUTES { - if !named_route_is_served(&state.settings, route.handler) { - continue; - } for method in route.primary_methods { router = router.route( route.path, diff --git a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs index 0a2c58136f..d900fe7524 100644 --- a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs @@ -208,55 +208,10 @@ fn proxy_route_reports_the_modules_geo_and_that_the_preparer_ran() { ); } -/// `[robots-txt] modules` naming the probe serves the rules the probe gives, -/// asked for the request's host and path and never its query, because the -/// answer is held and served to every crawler. -#[test] -fn robots_txt_selection_naming_a_module_serves_the_rules_it_gives() { - let settings = settings_with(&format!( - r#" - [robots-txt] - modules = ["testing.seam-probe", "allow_all"] - {HMAC_BLOCK} - {PROBE_BLOCK} - "# - )); - let router = router_with(settings, &[seam_probe::builder()]); - - let response = get(&router, "/robots.txt?crawler=one", None); - - assert_eq!( - response.status().as_u16(), - 200, - "the file should be served here and not by the origin" - ); - assert!( - response.headers().get("x-robots-tag").is_none(), - "a file made from contributors puts no blanket rule on responses" - ); - let file = body_text(response); - let agent = seam_probe::SEAM_PROBE_ROBOTS_AGENT; - assert!( - file.starts_with(&format!("User-Agent: {agent}\nDisallow: /")), - "the probe's rules should come first, as `modules` orders them: {file}" - ); - assert!( - file.contains("/robots.txt?\n"), - "the probe should be asked for the path with no query: {file}" - ); - assert!( - !file.contains("crawler=one"), - "nothing one crawler put in the address should reach a held answer: {file}" - ); - assert!( - file.ends_with("User-Agent: *\nAllow: /\n"), - "core's own rules should follow: {file}" - ); -} - -/// With `refuse_all` selected the file is the refusal, and every response -/// this adapter finalizes says not to index, the response of another route -/// included. +/// With `refuse_all` selected every response this adapter finalizes says +/// not to index, the response of another route included. The file itself is +/// the publisher path's, made by the `robots_txt` middleware the entry +/// names, so no route of this server names the path. #[test] fn refusing_every_crawler_tags_every_response() { let settings = settings_with(&format!( @@ -264,6 +219,11 @@ fn refusing_every_crawler_tags_every_response() { [robots-txt] modules = "refuse_all" always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] {HMAC_BLOCK} {PROBE_BLOCK} "# @@ -277,13 +237,12 @@ fn refusing_every_crawler_tags_every_response() { .map(str::to_owned) }; - let file = get(&router, "/robots.txt", None); - - assert_eq!(file.status().as_u16(), 200, "the refusal should be served"); - assert_eq!(tag(&file).as_deref(), Some("noindex, nofollow")); - assert_eq!( - body_text(file), - "User-agent: *\nAllow: /ads.txt\nDisallow: /\n" + assert!( + !router + .routes() + .into_iter() + .any(|route| route.path() == "/robots.txt"), + "the file is the publisher path's" ); let other = get(&router, seam_probe::SEAM_PROBE_REPORT_PATH, None); @@ -295,56 +254,20 @@ fn refusing_every_crawler_tags_every_response() { ); } -/// The methods this adapter registers for `/robots.txt`. -fn robots_txt_methods(router: &RouterService) -> Vec { - router - .routes() - .into_iter() - .filter(|route| route.path() == "/robots.txt") - .map(|route| route.method().to_string()) - .collect() -} - -/// With no `[robots-txt]` section the path is the publisher's, and the probe -/// being on offer changes nothing. With the section the path is this -/// server's for `GET` and `HEAD`. +/// Without the section no blanket rule is placed on responses. #[test] -fn robots_txt_is_this_servers_only_when_the_settings_carry_the_section() { +fn no_blanket_rule_is_placed_on_responses_without_the_section() { let without = router_with( settings_with(&format!("{HMAC_BLOCK}{PROBE_BLOCK}")), &[seam_probe::builder()], ); - assert_eq!( - robots_txt_methods(&without), - Vec::::new(), - "without the section no route of this server names the path" - ); let other = get(&without, seam_probe::SEAM_PROBE_REPORT_PATH, None); + assert!( other.headers().get("x-robots-tag").is_none(), "no blanket rule is placed on responses without the section" ); - - let with = router_with( - settings_with(&format!( - r#" - [robots-txt] - modules = "refuse_all" - {HMAC_BLOCK} - {PROBE_BLOCK} - "# - )), - &[seam_probe::builder()], - ); - - let methods = robots_txt_methods(&with); - for method in ["GET", "HEAD"] { - assert!( - methods.iter().any(|registered| registered == method), - "{method} /robots.txt should be answered here: {methods:?}" - ); - } } /// A request prepares exactly once whether a named route or the fallback diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 7d5cbea8ed..f45d67c6ae 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -48,7 +48,6 @@ use trusted_server_core::publisher::{ use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, }; -use trusted_server_core::robots_txt::{ROBOTS_TXT_PATH, handle_robots_txt}; use trusted_server_core::settings::Settings; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use trusted_server_core::settings_data::{default_config_key, default_secret_store_name}; @@ -1114,32 +1113,6 @@ fn build_router(state: &Arc) -> RouterService { } } - // `/robots.txt` is this server's only when the settings carry a - // `[robots-txt]` section. Without one the path reaches the publisher's - // origin like any other, so a publisher keeping their own file is - // left alone. - if state.settings.robots_txt.is_some() { - let s = Arc::clone(&state); - let robots_txt_handler = move |ctx: RequestContext| { - let s = Arc::clone(&s); - async move { - let services = s.services_for_request(&ctx); - Ok::( - handle_robots_txt(&s.settings, &services, &s.registry, ctx.into_request()) - .await - .unwrap_or_else(|e| http_error(&e)), - ) - } - }; - for method in publisher_fallback_methods() { - if matches!(method, Method::GET | Method::HEAD) { - builder = builder.route(ROBOTS_TXT_PATH, method, robots_txt_handler.clone()); - } else { - builder = builder.route(ROBOTS_TXT_PATH, method, fallback.clone()); - } - } - } - for method in publisher_fallback_methods() { builder = builder.route("/", method.clone(), fallback.clone()); builder = builder.route("/{*rest}", method, fallback.clone()); diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 28127614cd..7bae070ca4 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -935,6 +935,11 @@ fn router_refusing_every_crawler() -> RouterService { [robots-txt] modules = "refuse_all" always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] "#, ) .expect("should parse route test settings"); @@ -959,22 +964,19 @@ fn robots_tag(response: &Response) -> Option { .map(str::to_owned) } +/// The file itself is the publisher path's, made by the `robots_txt` +/// middleware the entry names, so no route of this server names the path. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn refusing_every_crawler_serves_the_refusal_and_tags_every_response() { - let file = route( - router_refusing_every_crawler(), - robots_request("GET", "/robots.txt"), - ) - .await; - - assert_eq!( - file.status().as_u16(), - 200, - "the refusal should be served here" +async fn refusing_every_crawler_tags_every_response_and_registers_no_route() { + let registered: Vec = router_refusing_every_crawler() + .routes() + .into_iter() + .map(|route| route.path().to_string()) + .collect(); + assert!( + !registered.iter().any(|path| path == "/robots.txt"), + "the file is the publisher path's: {registered:?}" ); - assert_eq!(robots_tag(&file).as_deref(), Some("noindex, nofollow")); - let body = file.into_body().into_bytes().unwrap_or_default(); - assert_eq!(&body[..], b"User-agent: *\nAllow: /ads.txt\nDisallow: /\n"); let other = route( router_refusing_every_crawler(), @@ -989,73 +991,6 @@ async fn refusing_every_crawler_serves_the_refusal_and_tags_every_response() { ); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_head_of_robots_txt_is_answered_here_with_no_body() { - let response = route( - router_refusing_every_crawler(), - robots_request("HEAD", "/robots.txt"), - ) - .await; - - assert_eq!( - response.status().as_u16(), - 200, - "a HEAD should be answered here as a GET is" - ); - assert_eq!( - response - .headers() - .get("content-type") - .and_then(|value| value.to_str().ok()), - Some("text/plain; charset=utf-8") - ); - let body = response.into_body().into_bytes().unwrap_or_default(); - assert!(body.is_empty(), "a HEAD carries no body"); -} - -/// A method that is neither `GET` nor `HEAD` is the publisher's, as it is on -/// any other path, so it is not answered with the file. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn another_method_on_robots_txt_is_not_answered_with_the_file() { - let response = route( - router_refusing_every_crawler(), - robots_request("POST", "/robots.txt"), - ) - .await; - - let body = response.into_body().into_bytes().unwrap_or_default(); - assert!( - !body.starts_with(b"User-agent:"), - "a POST should reach the publisher fallback and not be given the file" - ); -} - -#[test] -fn robots_txt_is_this_servers_only_when_the_settings_carry_the_section() { - let registered = |router: RouterService| -> Vec<(String, String)> { - router - .routes() - .into_iter() - .map(|route| (route.method().to_string(), route.path().to_string())) - .collect() - }; - - let without = registered(test_router()); - assert!( - !without.iter().any(|(_, path)| path == "/robots.txt"), - "without the section the path is the publisher's: {without:?}" - ); - - let with = registered(router_refusing_every_crawler()); - for method in ["GET", "HEAD"] { - assert!( - with.iter() - .any(|(registered, path)| registered == method && path == "/robots.txt"), - "{method} /robots.txt should be answered here: {with:?}" - ); - } -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_response_carries_no_blanket_rule_without_the_section() { let response = route( diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index fbacf0acfb..c768b0eec3 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1829,8 +1829,21 @@ impl IntegrationRegistry { inner.ec_module = resolve_ec_module(settings, &inner); inner.device_module = resolve_device_module(settings, &inner)?; inner.geo_module = geo_module; - check_middleware(settings, &inner)?; check_robots_txt_selection(settings, &inner)?; + if let Some(config) = settings.robots_txt.as_ref() { + // Core's own middleware for the file, made here because the + // contributors it asks are known only once every registration + // is in, and a middleware cannot reach the registry. + let selected = crate::robots_txt::select(config, &inner.robots_txt_contributors)?; + inner.middleware.push(( + crate::robots_txt::middleware::MODULE, + Arc::new(crate::robots_txt::middleware::RobotsTxt::new( + config.clone(), + selected, + )) as Arc, + )); + } + check_middleware(settings, &inner)?; Ok(Self { inner: Arc::new(inner), diff --git a/crates/trusted-server-core/src/module_context.rs b/crates/trusted-server-core/src/module_context.rs index 7a3595bfda..958c69892e 100644 --- a/crates/trusted-server-core/src/module_context.rs +++ b/crates/trusted-server-core/src/module_context.rs @@ -165,6 +165,18 @@ impl ResolvedRequest { } } + /// The method, address, host and scheme `view` carries. + #[must_use] + pub fn from_view(view: &ModuleRequest<'_>) -> Self { + Self { + method: view.method.clone(), + path: view.path.to_owned(), + query: view.query.to_owned(), + host: view.host.to_owned(), + scheme: view.scheme.to_owned(), + } + } + /// The same request with no query, for an answer that is held and served /// to every caller, which must not depend on what one caller put in the /// address. diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 523627aa8b..ee3f63053e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -5579,8 +5579,9 @@ pub async fn handle_publisher_request( request_scheme, &request_path, ); - let module_context = - ModuleContext::new(module_request).with_request_state(ec_context, services); + let module_context = ModuleContext::new(module_request) + .with_settings(settings) + .with_request_state(ec_context, services); let stored_origin_host = settings.publisher.origin_host(); let document = TextDocument { media_type: crate::middleware::HTML_MEDIA_TYPE.to_owned(), @@ -5917,8 +5918,9 @@ pub async fn handle_publisher_request( }; let module_request = ModuleRequest::new(&request_method, request_host, request_scheme, &request_path); - let module_context = - ModuleContext::new(module_request).with_request_state(ec_context, services); + let module_context = ModuleContext::new(module_request) + .with_settings(settings) + .with_request_state(ec_context, services); let document = TextDocument { media_type: media_type_of(&content_type), request_path: &request_path, @@ -21140,12 +21142,33 @@ mod tests { /// As [`proxy`], with the settings given. async fn proxy_with(settings: Settings, method: Method, path: &str, status: u16) -> Proxied { + proxy_origin( + settings, + method, + path, + status, + b"the origin's page", + "text/html; charset=utf-8", + ) + .await + } + + /// As [`proxy_with`], with the origin answering `status` with `body` as + /// `content_type`. + async fn proxy_origin( + settings: Settings, + method: Method, + path: &str, + status: u16, + body: &[u8], + content_type: &'static str, + ) -> Proxied { let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( status, - b"the origin's page".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], + body.to_vec(), + vec![("content-type", content_type)], ); let services = build_services_with_http_client( Arc::clone(&stub) as Arc @@ -21222,6 +21245,93 @@ mod tests { .and_then(|value| value.to_str().ok()) } + /// Settings refusing every crawler, with the entry that places core's + /// robots.txt middleware on the file. + fn settings_refusing_every_crawler() -> Settings { + use crate::robots_txt::middleware as robots_txt; + + let mut settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: robots_txt::MEDIA_TYPE.to_owned(), + path: Some(crate::robots_txt::ROBOTS_TXT_PATH.to_owned()), + middleware: vec![robots_txt::MODULE.to_owned()], + }]); + let mut refusing = crate::robots_txt::RobotsTxtConfig::refuse_all(); + refusing.always_allow = vec!["/ads.txt".to_owned()]; + settings.robots_txt = Some(refusing); + settings + } + + /// The file a proxied robots.txt request streams, with its status and + /// content type checked. + async fn streamed_robots_txt(proxied: Proxied) -> String { + let PublisherResponse::Stream { + response, + body, + params, + } = proxied.response + else { + panic!("the file should stream through the entry's middleware"); + }; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + content_type_of(&response), + Some("text/plain; charset=utf-8"), + "should be a document of the entry's media type" + ); + let mut params = *params; + let mut output = Vec::new(); + stream_publisher_body_async( + body, + &mut output, + &mut params, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should stream the file"); + String::from_utf8(output).expect("the file is text") + } + + /// The file the middleware makes replaces the origin's own. + #[tokio::test] + async fn robots_txt_replaces_the_origins_file() { + let proxied = proxy_origin( + settings_refusing_every_crawler(), + Method::GET, + crate::robots_txt::ROBOTS_TXT_PATH, + 200, + b"User-agent: *\nAllow: /\n", + "text/plain; charset=utf-8", + ) + .await; + + assert_eq!( + streamed_robots_txt(proxied).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the refusal replaces the origin's file" + ); + } + + /// Where the origin has no file, the creating rule makes it. + #[tokio::test] + async fn robots_txt_is_made_where_the_origin_has_none() { + let proxied = proxy_with( + settings_refusing_every_crawler(), + Method::GET, + crate::robots_txt::ROBOTS_TXT_PATH, + 404, + ) + .await; + + assert_eq!( + streamed_robots_txt(proxied).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the file is made" + ); + } + #[tokio::test] async fn a_file_the_origin_does_not_have_is_made_by_the_middleware_named_for_its_path() { use crate::integrations::registry_test_support::middleware_fixture as fixture; diff --git a/crates/trusted-server-core/src/robots_txt/middleware.rs b/crates/trusted-server-core/src/robots_txt/middleware.rs new file mode 100644 index 0000000000..a92427163a --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/middleware.rs @@ -0,0 +1,229 @@ +//! `/robots.txt` as a middleware of core's own, named `robots_txt` in a +//! `[[fetch]]` or `[[serve]]` entry on the file's path. Its preparation makes +//! the file from the selected contributors, through their holding, and its +//! action replaces whatever the origin sent with that file. Where the origin +//! has no file, the creating rule runs it on an empty one. + +use std::any::Any; +use std::time::Duration; + +use error_stack::Report; + +use super::{RETRY_AFTER_SECONDS, RobotsTxtConfig, Selected, document, file_for}; +use crate::error::TrustedServerError; +use crate::middleware::{ + Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, Preparation, Refusal, +}; +use crate::module_context::{ModuleCall, ModuleRequest, ResolvedRequest}; +use crate::platform::RuntimeServices; +use crate::settings::Settings; +use crate::streaming_processor::StreamProcessor; + +/// The name an entry writes, and the integration the registry lists the +/// middleware under. +pub const MODULE: &str = "robots_txt"; + +/// The media type the middleware works on. +pub const MEDIA_TYPE: &str = "text/plain"; + +/// What a request is told when no file can be made. +const UNAVAILABLE: &str = "robots.txt is temporarily unavailable\n"; + +/// The middleware, holding the section it was made for and the contributors +/// the registry resolved for it, in the order `modules` names them. +#[derive(Debug)] +pub struct RobotsTxt { + config: RobotsTxtConfig, + selected: Vec, +} + +impl RobotsTxt { + /// The middleware for `config`, with `selected` the contributors in the + /// order `modules` names them. + #[must_use] + pub(crate) fn new(config: RobotsTxtConfig, selected: Vec) -> Self { + Self { config, selected } + } + + /// The file for the request, which is the refusal alone or every selected + /// contributor's held answer assembled. + async fn file( + &self, + request: ModuleRequest<'_>, + settings: &Settings, + services: &RuntimeServices, + ) -> Result> { + if self.config.refuses_all() { + return Ok(document::refusal(&self.config.always_allow)); + } + // The answers are held and served to every crawler, so the + // contributors are handed the request with nothing one crawler put in + // the address. + let request = ResolvedRequest::from_view(&request).without_query(); + file_for(&self.config, &self.selected, &request, services, settings).await + } +} + +impl Middleware for RobotsTxt { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + _context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async move { + let file = match call.inject(self, Self::file) { + Ok(file) => file.await, + Err(withheld) => Err(Report::new(TrustedServerError::Configuration { + message: format!("robots.txt cannot be made for this request: {withheld:?}"), + })), + }; + match file { + Ok(file) => Ok(Box::new(file) as Box), + Err(report) => { + log::error!("Could not assemble robots.txt: {report:?}"); + Err(Refusal::service_unavailable( + Duration::from_secs(u64::from(RETRY_AFTER_SECONDS)), + UNAVAILABLE, + )) + } + } + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + match context.prepared.take::(MODULE) { + Some(file) => MiddlewareAction { + stream: Some(Box::new(ReplaceBody { file: Some(file) })), + ..MiddlewareAction::pass() + }, + // Nothing prepared means the preparation did not run, which no + // request can cause. The origin's own file is left as it is, + // because an empty file would tell crawlers to crawl everything. + None => { + log::error!("robots.txt was not prepared, so the origin's file is served as it is"); + MiddlewareAction::pass() + } + } + } +} + +/// Drops the body it is given and writes the file in its place. +struct ReplaceBody { + file: Option, +} + +impl StreamProcessor for ReplaceBody { + fn process_chunk(&mut self, _chunk: &[u8], is_last: bool) -> Result, std::io::Error> { + Ok(if is_last { + self.file.take().unwrap_or_default().into_bytes() + } else { + Vec::new() + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::integrations::IntegrationDocumentState; + use crate::middleware::test_support::context; + use crate::middleware::{MiddlewareChain, Prepared}; + + fn middleware() -> RobotsTxt { + RobotsTxt::new(RobotsTxtConfig::refuse_all(), Vec::new()) + } + + /// What the middleware is told in `phase`, with `prepared` as what its + /// preparation left. + fn context_with<'a>( + phase: MiddlewarePhase, + state: &'a IntegrationDocumentState, + prepared: &'a Prepared, + ) -> MiddlewareContext<'a> { + MiddlewareContext { + prepared, + ..context(phase, state) + } + } + + #[test] + fn the_middleware_works_on_text_in_either_phase() { + let middleware = middleware(); + + assert!(middleware.handles_media_type(MEDIA_TYPE)); + assert!(!middleware.handles_media_type(crate::middleware::HTML_MEDIA_TYPE)); + assert_eq!(middleware.phases(), &MiddlewarePhase::ALL); + assert_eq!(middleware.middleware_id(), MODULE); + } + + #[test] + fn the_prepared_file_replaces_whatever_the_origin_sent() { + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + prepared.insert( + MODULE, + Box::new(String::from("User-agent: *\nDisallow: /\n")), + ); + let context = context_with(MiddlewarePhase::Fetch, &state, &prepared); + + let mut processor = middleware() + .create(&context) + .stream + .expect("a prepared file should be written as a stream processor"); + let mut output = processor + .process_chunk(b"User-agent: *\nAllow: /\n", false) + .expect("should take the origin's file"); + output.extend( + processor + .process_chunk(b"", true) + .expect("should write the file at the end"), + ); + + assert_eq!(output, b"User-agent: *\nDisallow: /\n"); + } + + #[test] + fn with_nothing_prepared_the_origins_file_is_left_as_it_is() { + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = context_with(MiddlewarePhase::Fetch, &state, &prepared); + + let action = middleware().create(&context); + + assert!(action.stream.is_none(), "should pass the body through"); + assert!(action.head_inserts.is_empty()); + } + + #[test] + fn a_value_of_another_type_under_the_name_is_not_a_file() { + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + prepared.insert(MODULE, Box::new(7_u32)); + let context = context_with(MiddlewarePhase::Serve, &state, &prepared); + + assert!(middleware().create(&context).stream.is_none()); + } + + #[test] + fn a_chain_plans_the_middleware_on_text() { + let chain = MiddlewareChain::new( + MiddlewarePhase::Fetch, + MEDIA_TYPE, + vec![std::sync::Arc::new(middleware()) as std::sync::Arc], + ); + + assert_eq!(chain.ids(), [MODULE]); + } +} diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs index e6b8f52e95..d0a72c2b21 100644 --- a/crates/trusted-server-core/src/robots_txt/mod.rs +++ b/crates/trusted-server-core/src/robots_txt/mod.rs @@ -38,14 +38,15 @@ mod contributor; pub mod document; mod holding; +pub mod middleware; use std::collections::BTreeMap; use std::sync::Arc; use edgezero_core::body::Body as EdgeBody; use error_stack::Report; +use http::Response; use http::header::{self, HeaderValue}; -use http::{Method, Request, Response, StatusCode}; use serde::{Deserialize, Deserializer, Serialize}; use validator::{Validate, ValidationError}; @@ -53,7 +54,6 @@ pub use contributor::{ALLOW_ALL, REFUSE_ALL, RobotsTxtContributor}; pub use document::{Contribution, Group, Record}; use crate::error::TrustedServerError; -use crate::integrations::IntegrationRegistry; use crate::module_context::ResolvedRequest; use crate::platform::RuntimeServices; use crate::settings::Settings; @@ -61,9 +61,6 @@ use crate::settings::Settings; /// The path this module answers. pub const ROBOTS_TXT_PATH: &str = "/robots.txt"; -/// How long a browser or an edge cache may hold the answer. -const CLIENT_CACHE_SECONDS: u32 = 3600; - /// How long a crawler is asked to wait when there is no file to give it. const RETRY_AFTER_SECONDS: u32 = 600; @@ -294,77 +291,48 @@ pub fn apply_response_robots_tag(settings: &Settings, response: &mut Response>); + +/// The entries of `config.modules` resolved against `contributors`, the +/// contributors a deployment runs by the names they were declared under. /// /// # Errors /// -/// When the settings carry no `[robots-txt]` section. An adapter registers -/// this route only when the section is present, so that is a wiring fault and -/// not something a request can cause. -pub async fn handle_robots_txt( - settings: &Settings, - services: &RuntimeServices, - registry: &IntegrationRegistry, - req: Request, -) -> Result, Report> { - let response = answer(settings, services, registry, &req).await?; - Ok(if req.method() == Method::HEAD { - response.map(|_| EdgeBody::empty()) - } else { - response - }) -} - -async fn answer( - settings: &Settings, - services: &RuntimeServices, - registry: &IntegrationRegistry, - req: &Request, -) -> Result, Report> { - let config = settings.robots_txt.as_ref().ok_or_else(|| { - Report::new(configuration( - "the robots.txt route is registered but there is no [robots-txt] section".to_owned(), - )) - })?; - // A refusal answers without asking anything, before any contributor, - // held answer or network is involved. That is the point of it, because a - // site that is not ready to be crawled can then say so with nothing - // provisioned at all. - if config.refuses_all() { - return Ok(respond(Ok(document::refusal(&config.always_allow)))); - } - let mut selected: Vec = Vec::new(); - for id in &config.modules { - if id == ALLOW_ALL { +/// When an entry names a contributor `contributors` does not carry, which +/// the registry refuses before it asks, so this is a wiring fault. +pub(crate) fn select( + config: &RobotsTxtConfig, + contributors: &[(&'static str, Arc)], +) -> Result, Report> { + let mut selected = Vec::with_capacity(config.modules.len()); + for written in &config.modules { + if written == REFUSE_ALL { + // The refusal needs no contributor, and the middleware answers it + // before asking any. + continue; + } + if written == ALLOW_ALL { selected.push((ALLOW_ALL, None)); continue; } - let Some((declared, contributor)) = registry.robots_txt_contributor(id) else { - // The registry refuses a selection nothing supplies when it is - // built, so this is a wiring fault, and it still fails closed. - return Ok(respond(Err(Report::new(configuration(format!( - "[robots-txt] modules names `{id}`, which no module in this deployment supplies" - )))))); + let Some((declared, contributor)) = contributors.iter().find(|(name, _)| { + *name == written || crate::module_name::short_form(MODULE_TYPE, name) == written + }) else { + return Err(Report::new(configuration(format!( + "[robots-txt] modules names `{written}`, which no module in this deployment supplies" + )))); }; - selected.push((declared, Some(contributor))); - } - // The answers are held and served to every crawler, so the contributors - // are handed the request with nothing one crawler put in the address. - let request = ResolvedRequest::of(req, services.client_info()).without_query(); - Ok(respond( - file_for(config, &selected, &request, services, settings).await, - )) + selected.push((declared, Some(Arc::clone(contributor)))); + } + Ok(selected) } -/// One entry of `modules` as the handler resolved it: the name a contributor -/// was declared under and the contributor, or `allow_all` and none. -type Selected = (&'static str, Option>); - /// The file for `config`, from the contributions of `selected` in order, /// each asked for `request`. -async fn file_for( +pub(crate) async fn file_for( config: &RobotsTxtConfig, selected: &[Selected], request: &ResolvedRequest, @@ -383,38 +351,6 @@ async fn file_for( Ok(document::assemble(config, &contributions)) } -fn respond(file: Result>) -> Response { - match file { - Ok(file) => Response::builder() - .status(StatusCode::OK) - .header( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ) - .header( - header::CACHE_CONTROL, - format!("public, max-age={CLIENT_CACHE_SECONDS}"), - ) - .body(EdgeBody::from(file.into_bytes())) - .expect("should build the robots.txt response"), - Err(report) => { - log::error!("Could not assemble robots.txt: {report:?}"); - Response::builder() - .status(StatusCode::SERVICE_UNAVAILABLE) - .header( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ) - .header(header::RETRY_AFTER, RETRY_AFTER_SECONDS.to_string()) - .header(header::CACHE_CONTROL, HeaderValue::from_static("no-store")) - .body(EdgeBody::from( - b"robots.txt is temporarily unavailable\n".to_vec(), - )) - .expect("should build the robots.txt failure response") - } - } -} - #[cfg(test)] mod tests { use std::sync::Arc; @@ -424,7 +360,10 @@ mod tests { settings, }; use super::*; - use crate::integrations::{IntegrationBuilder, IntegrationRegistration}; + use crate::integrations::{IntegrationBuilder, IntegrationRegistration, IntegrationRegistry}; + use crate::middleware::Middleware; + use crate::module_context::{ModuleContext, ModuleRequest}; + use http::StatusCode; fn parsed(document: serde_json::Value) -> RobotsTxtConfig { serde_json::from_value(document).expect("should read the robots.txt settings") @@ -615,71 +554,122 @@ mod tests { ); } + /// What the middleware prepares for `settings`, with the services given, + /// through the registry that made it from `[robots-txt]`, as the handler + /// runs it for `/robots.txt` in the fetch phase. + async fn prepared_by( + settings: &Settings, + extra: &[IntegrationBuilder], + services: &RuntimeServices, + ) -> Result, crate::middleware::Refusal> { + use crate::middleware::{MiddlewarePhase, Prepared}; + + let registry = IntegrationRegistry::with_registrations(settings, extra) + .expect("should build the registry"); + let chain = registry.middleware_chain( + &settings.fetch, + MiddlewarePhase::Fetch, + middleware::MEDIA_TYPE, + ROBOTS_TXT_PATH, + ); + assert_eq!( + chain.ids(), + [middleware::MODULE], + "the entry names the middleware" + ); + let request = robots_request(); + let module_context = + ModuleContext::new(ModuleRequest::of(&request, services.client_info())) + .with_settings(settings) + .with_services(services); + let state = crate::integrations::IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = crate::middleware::MiddlewareContext { + prepared: &prepared, + ..crate::middleware::test_support::context(MiddlewarePhase::Fetch, &state) + }; + chain.prepare(&module_context, &context).await?; + Ok(prepared.take::(middleware::MODULE)) + } + + /// `settings` with `config` as its `[robots-txt]` section and the entry + /// that places the middleware on the file. + fn settings_placing(config: RobotsTxtConfig) -> Settings { + let mut settings = settings(); + settings.robots_txt = Some(config); + settings.fetch = + crate::middleware::PhaseEntries::new(vec![crate::middleware::PhaseEntry { + media_type: middleware::MEDIA_TYPE.to_owned(), + path: Some(ROBOTS_TXT_PATH.to_owned()), + middleware: vec![middleware::MODULE.to_owned()], + }]); + settings + } + /// Leaving out the rules of a contributor that failed would allow by /// omission what those rules refused, so the whole file fails closed. #[tokio::test] - async fn with_nothing_held_a_failure_is_service_unavailable() { + async fn with_nothing_held_a_failure_refuses_the_request() { let failing: Arc = Arc::new(Scripted::new(vec![Err("cannot answer".to_owned())])); let config = parsed(serde_json::json!({ "modules": ["failing", "allow_all"] })); + let middleware = + middleware::RobotsTxt::new(config, vec![("failing", Some(failing)), (ALLOW_ALL, None)]); + let settings = settings(); + let services = services_with(Arc::default(), Arc::default()); + let request = robots_request(); + let module_context = + ModuleContext::new(ModuleRequest::of(&request, services.client_info())) + .with_settings(&settings) + .with_services(&services); + let state = crate::integrations::IntegrationDocumentState::default(); + let context = crate::middleware::test_support::context( + crate::middleware::MiddlewarePhase::Fetch, + &state, + ); - let response = respond( - file_for( - &config, - &[("failing", Some(failing)), (ALLOW_ALL, None)], - &request(), - &services_with(Arc::default(), Arc::default()), - &settings(), + let refusal = middleware + .prepare( + module_context.call( + middleware::MODULE, + crate::permissions::PermissionSet::none(), + ), + &context, ) - .await, - ); + .expect("the middleware prepares") + .await + .expect_err("a crawler reads an empty or allow everything file as permission"); + assert_eq!(refusal.status, StatusCode::SERVICE_UNAVAILABLE); assert_eq!( - response.status(), - StatusCode::SERVICE_UNAVAILABLE, - "a crawler reads an empty or allow everything file as permission" - ); - assert!(response.headers().contains_key(header::RETRY_AFTER)); - assert_eq!( - response.headers().get(header::CACHE_CONTROL), - Some(&HeaderValue::from_static("no-store")), - "a failure must not be held in place of the file" + refusal.retry_after, + Some(std::time::Duration::from_secs(u64::from( + RETRY_AFTER_SECONDS + ))) ); + assert_eq!(refusal.message, "robots.txt is temporarily unavailable\n"); } /// The refusal asks nothing of anything, and dominates whatever else is /// selected, because a refusal cannot be loosened. #[tokio::test] async fn refusing_every_crawler_asks_no_contributor_and_reads_no_store() { - let mut settings = settings(); let mut refusing = parsed(serde_json::json!({ "modules": ["allow_all", "refuse_all"] })); refusing.always_allow = vec!["/ads.txt".to_owned()]; - settings.robots_txt = Some(refusing); + let settings = settings_placing(refusing); let store = Arc::new(MemoryKvStore::default()); - let registry = IntegrationRegistry::new(&settings).expect("should build the registry"); - let response = handle_robots_txt( + let file = prepared_by( &settings, + &[], &services_with(Arc::default(), Arc::clone(&store)), - ®istry, - robots_request(), ) .await - .expect("the refusal answers without asking"); + .expect("the refusal answers without asking") + .expect("the refusal is prepared"); - assert_eq!(response.status(), StatusCode::OK); assert_eq!( - response.headers().get(header::CACHE_CONTROL), - Some(&HeaderValue::from_static("public, max-age=3600")) - ); - let body = response - .into_body() - .into_bytes_bounded(1024) - .await - .expect("should read the body"); - assert_eq!( - body.as_ref(), - b"User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + file, "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", "the refusal, whatever else is selected" ); assert!( @@ -692,36 +682,24 @@ mod tests { ); } - /// A crawler that asks with `HEAD` is told what a `GET` would be told and - /// is sent no file. - #[tokio::test] - async fn a_head_is_answered_with_the_headers_and_no_body() { - let mut settings = settings(); - settings.robots_txt = Some(RobotsTxtConfig::refuse_all()); - let registry = IntegrationRegistry::new(&settings).expect("should build the registry"); - let mut request = robots_request(); - *request.method_mut() = Method::HEAD; - - let response = handle_robots_txt( - &settings, - &services_with(Arc::default(), Arc::default()), - ®istry, - request, - ) - .await - .expect("should answer"); + /// The registry makes the middleware only for settings that carry the + /// section, so a publisher keeping their own file is left alone. + #[test] + fn the_middleware_exists_only_with_the_section() { + let without = IntegrationRegistry::new(&settings()).expect("should build the registry"); + assert!( + !without.middleware_ids().contains(&middleware::MODULE), + "{:?}", + without.middleware_ids() + ); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - response.headers().get(header::CONTENT_TYPE), - Some(&HeaderValue::from_static("text/plain; charset=utf-8")) + let with = IntegrationRegistry::new(&settings_placing(RobotsTxtConfig::refuse_all())) + .expect("should build the registry"); + assert!( + with.middleware_ids().contains(&middleware::MODULE), + "{:?}", + with.middleware_ids() ); - let body = response - .into_body() - .into_bytes_bounded(16) - .await - .expect("should read the body"); - assert!(body.is_empty(), "a HEAD carries no body"); } /// The id of the builder that registers the echoing contributor. @@ -758,31 +736,23 @@ mod tests { .with_module_name(ECHO_BUILDER_MODULE) } - /// The handler asks a contributor for the request it is answering, being - /// its host and path, without the query and without the reader's + /// The middleware asks a contributor for the request it is answering, + /// being its host and path, without the query and without the reader's /// evidence. #[tokio::test] - async fn the_handler_asks_the_contributors_for_the_request_it_answers() { - let settings = settings_selecting_echo(); - let registry = IntegrationRegistry::with_registrations(&settings, &[echo_builder()]) - .expect("should build the registry"); + async fn the_middleware_asks_the_contributors_for_the_request_it_answers() { + let mut settings = settings_selecting_echo(); + settings.fetch = settings_placing(RobotsTxtConfig::refuse_all()).fetch; - let response = handle_robots_txt( + let file = prepared_by( &settings, + &[echo_builder()], &services_with(Arc::default(), Arc::default()), - ®istry, - robots_request(), ) .await - .expect("should answer"); + .expect("should answer") + .expect("the file is prepared"); - assert_eq!(response.status(), StatusCode::OK); - let body = response - .into_body() - .into_bytes_bounded(4096) - .await - .expect("should read the body"); - let file = String::from_utf8(body.to_vec()).expect("the file is text"); let config = settings .robots_txt .as_ref() diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 9d98899e2f..aef91fc5f5 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -4290,6 +4290,7 @@ impl Settings { self.validate_asset_image_optimizer_profile_sets()?; if let Some(robots_txt) = &self.robots_txt { robots_txt.prepare_runtime()?; + self.require_robots_txt_placed()?; } if let Some(co) = &mut self.creative_opportunities { @@ -4327,6 +4328,30 @@ impl Settings { Ok(()) } + /// Refuses a `[robots-txt]` section that no `[[fetch]]` or `[[serve]]` + /// entry for `text/plain` names the `robots_txt` middleware for, because + /// the file would never be made. + fn require_robots_txt_placed(&self) -> Result<(), Report> { + use crate::robots_txt::middleware::{MEDIA_TYPE, MODULE}; + + let placed = [&self.fetch, &self.serve].iter().any(|entries| { + entries.entries().iter().any(|entry| { + entry.media_type == MEDIA_TYPE && entry.middleware.iter().any(|name| name == MODULE) + }) + }); + if placed { + return Ok(()); + } + Err(Report::new(TrustedServerError::Settings { + message: format!( + "[robots-txt] is set, but no [[fetch]] or [[serve]] entry for `{MEDIA_TYPE}` \ + names `{MODULE}`, so the file would never be made. Add an entry with \ + media_type = `{MEDIA_TYPE}`, path = `{}` and the middleware", + crate::robots_txt::ROBOTS_TXT_PATH + ), + })) + } + /// Returns compiled creative opportunity slots when template delivery is enabled. #[must_use] pub fn creative_opportunity_slots( @@ -5086,7 +5111,26 @@ mod tests { } fn settings_str_with_robots_txt(section: &str) -> String { - format!("{}\n[robots-txt]\n{section}\n", crate_test_settings_str()) + format!( + "{}\n[robots-txt]\n{section}\n\n[[fetch]]\nmedia_type = \"text/plain\"\npath = \"/robots.txt\"\nmiddleware = [\"robots_txt\"]\n", + crate_test_settings_str() + ) + } + + #[test] + fn a_robots_txt_section_with_no_entry_placing_the_middleware_is_refused() { + let toml = format!( + "{}\n[robots-txt]\nmodules = \"refuse_all\"\n", + crate_test_settings_str() + ); + + let error = Settings::from_toml(&toml).expect_err("the file would never be made"); + + let message = format!("{error:?}"); + assert!( + message.contains("no [[fetch]] or [[serve]] entry for `text/plain` names `robots_txt`"), + "{message}" + ); } #[test] diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index ff9703e551..dfbaadaecc 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -110,6 +110,7 @@ pub fn middleware() -> Vec { use trusted_server_cmp_inmobi as inmobi; use trusted_server_cmp_sourcepoint as sourcepoint; use trusted_server_core::integrations::js_asset_proxy; + use trusted_server_core::robots_txt::middleware as robots_txt; use trusted_server_framework_nextjs as nextjs; use trusted_server_identity_lockr as lockr; use trusted_server_tag_google_tag_manager as google_tag_manager; @@ -151,6 +152,14 @@ pub fn middleware() -> Vec { media_type: ads_txt_sellers::MEDIA_TYPE, name: ads_txt_sellers::MODULE, }, + // Core makes the file `[robots-txt]` describes under the module's + // own name, on the file's path alone. + StockMiddleware { + integration: robots_txt::MODULE, + phase: Fetch, + media_type: robots_txt::MEDIA_TYPE, + name: robots_txt::MODULE, + }, ] } @@ -250,6 +259,10 @@ mod tests { checked.extend(running); } + // Core's robots.txt middleware is registered by the section, not by a + // module, and `core_s_robots_txt_middleware_is_registered_with_the_section` + // checks it. + checked.push(trusted_server_core::robots_txt::middleware::MODULE); for row in &listed { assert!( checked.contains(&row.integration), @@ -261,6 +274,38 @@ mod tests { } } + /// The row for core's robots.txt middleware is registered by settings + /// that carry the section and the entry, in the phase the row lists. + #[test] + fn core_s_robots_txt_middleware_is_registered_with_the_section() { + use trusted_server_core::robots_txt::middleware as robots_txt; + + let settings = trusted_server_core::html_processor::test_support::page_settings( + "[robots-txt] +modules = \"refuse_all\" + +[[fetch]] +media_type = \"text/plain\" +\n path = \"/robots.txt\" +middleware = [\"robots_txt\"] +", + ); + let registry = IntegrationRegistry::with_registrations(&settings, &builders()) + .expect("should build a registry with the section"); + let row = middleware() + .into_iter() + .find(|row| row.name == robots_txt::MODULE) + .expect("the stock list should carry the row"); + + assert!( + registry + .middleware_in(row.phase) + .contains(&robots_txt::MODULE), + "the section should register the middleware for the listed phase" + ); + assert_eq!(row.media_type, robots_txt::MEDIA_TYPE); + } + #[test] fn the_listed_middleware_follow_the_order_the_modules_register_in() { // What registers from the auction plan goes first, core's own module @@ -274,6 +319,9 @@ mod tests { order.push(builder.id()); } } + // Core makes the file's middleware once every registration is in, + // so it comes after every module's. + order.push(trusted_server_core::robots_txt::middleware::MODULE); let place = |integration: &str| { order .iter() @@ -327,19 +375,30 @@ mod tests { .iter() .filter(|entry| entry.media_type == media_type) .collect(); - assert_eq!( - covering.len(), - 1, - "should document one [[{phase}]] entry for {media_type}" - ); if media_type == HTML_MEDIA_TYPE { + assert_eq!( + covering.len(), + 1, + "should document one [[{phase}]] entry for {media_type}" + ); assert!( covering[0].path.is_none(), "should document the [[{phase}]] entry that covers every page" ); + } else { + // A file's middleware is placed on the file's own path, one + // entry for each file. + assert!( + !covering.is_empty() && covering.iter().all(|entry| entry.path.is_some()), + "should document a [[{phase}]] entry on a path for each {media_type} file" + ); } + let named: Vec<&str> = covering + .iter() + .flat_map(|entry| entry.middleware.iter().map(String::as_str)) + .collect(); assert_eq!( - covering[0].middleware, listed, + named, listed, "should document every {phase} middleware the stock modules supply for \ {media_type}, in the listed order" ); diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 565bedb857..b544b80330 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -199,7 +199,9 @@ pull_sync_concurrency = 3 # # A module's own settings go in its own block, [robots-txt.]. # A publisher who sells advertising wants /ads.txt in always_allow, because -# a crawler may ignore an ads.txt that robots.txt disallows. +# a crawler may ignore an ads.txt that robots.txt disallows. The file is made +# by core's robots_txt middleware where a [[fetch]] entry for text/plain on +# the file's path names it, see the entry below the ads.txt one. # [robots-txt] # modules = ["refuse_all"] # always_allow = ["/ads.txt"] @@ -220,6 +222,14 @@ pull_sync_concurrency = 3 # path = "/ads.txt" # middleware = ["ads-txt.sellers"] +# The robots.txt the [robots-txt] section describes, made by core's own +# middleware where this entry names it. The file replaces the origin's own +# and is made where the origin has none. +# [[fetch]] +# media_type = "text/plain" +# path = "/robots.txt" +# middleware = ["robots_txt"] + # Sign outbound OpenRTB/API requests. The runtime reads its keys from fixed # store names (`jwks_store` / `signing_keys`), and `ts keys` writes them. To keep # signing OFF, leave this whole block commented out. From 71dbb17558507b746864e96f8b87cda8a1224f4a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 11 Oct 2026 11:23:41 +0100 Subject: [PATCH 02/23] The guides say robots.txt is made by its middleware The configuration guide's robots.txt section says the file is made by the `robots_txt` middleware where an entry names it, in place of the origin's file or where the origin has none, that a section no entry places is refused, and that the response takes the deployment's cache policy for the path. Its example carries the entry, and the placing table has the middleware's row. The changelog records the middleware and the route's removal. --- CHANGELOG.md | 2 ++ docs/guide/configuration.md | 11 +++++++++-- docs/guide/integration-guide.md | 18 +++++++++--------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b928cf777f..c0f61e91c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed +- The `/robots.txt` route on the four adapters, with `handle_robots_txt` and the route's own `Cache-Control: public, max-age=3600`. The file is made by the `robots_txt` middleware where an entry names it, and takes the deployment's cache policy for the path as every file does. - **Breaking:** HTTP Basic authentication is removed, with the `[[handlers]]` list that configured it. No request is asked for a password, and nothing is served beneath `/_ts/admin`. A configuration that still carries `[[handlers]]` is refused, with directions, when it is validated and when the service starts, so that no deployment believes a path is protected when it is not. A build before this change refuses a configuration without a handler covering `/_ts/admin`, and one with `[request_signing]` and no store ids, so no one configuration starts both builds: the new configuration and the new build go live together. A staged deploy reads a configuration key of its own (`ts deploy --staging`, `ts config push --staging`), which lets the pair be checked together first. Put a password on a path of the publisher's own at its origin or in the hosting platform. The `handler_password` secret is no longer read, and its entries in `fastly.toml`, the Spin manifest and the smoke scripts are removed. - **Breaking:** `POST /_ts/admin/cache/purge` is removed. It let anyone holding one shared password empty the service's cache over the publisher's own domain. Purge with `ts cache purge`, which asks the hosting platform. A request to the removed path is answered `404` by the server itself and is never forwarded to the origin. - The Fastly adapter's management API client is removed. The deployed service no longer reads a Fastly API token from a secret store named `api-keys`, which it used to write its own config store and secret store when a key was rotated. A deployment that linked that store can unlink it and revoke the token. `fastly.toml` and the Viceroy template drop their `api-keys` entries. @@ -80,6 +81,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `robots_txt`, core's own middleware for the file `[robots-txt]` describes, named in a `[[fetch]]` or `[[serve]]` entry for `text/plain` on `/robots.txt`. Its preparation makes the file from the selected modules through their holding, and its action replaces whatever the origin sent with it, so the file replaces the origin's own and the creating rule makes it where the origin has none. A module that cannot answer with nothing held refuses the request with `503` and `Retry-After` before any document is processed. A section with no entry naming the middleware is refused when the settings load. - A middleware may prepare, and may refuse. `Middleware::prepare` runs in the request's handler before the document is processed, as the middleware's module with the request's services, and what it answers is handed to `create` under the middleware's name, so a decision that needs a store or the network is made there while `create` stays synchronous. A refusal answers the request with its status and `Retry-After`, with no document, and nothing is stored. - A loading cache. `PlatformLoadingCache` in core loads a missing value once per key however many requests ask for it at the same moment, gives every waiting request the value or the error, and holds nothing that failed. A crate offers one through `IntegrationBuilder::with_loading_cache`, each feature that holds answers selects the module whose cache holds them in its own `[cache.]` table, and an adapter takes the selections with `loading_caches_from`. A feature that selects none gets `NoLoadingCache`, which loads every time. The new `crates/loading-cache/51degrees`, the module `loading-cache.51degrees`, implements it with the published `fiftyone-caching` package as a brief copy in process memory over the host's value cache or key-value store, and the stock list ships it, so every adapter takes it. A load that may finish apart from its request runs on a pool of threads on Axum and as a task of its own on Cloudflare, where it keeps running after the response only when the Worker's handling of the request is wrapped in the crate's `scope_request`, and every other load runs in the caller. - `[cache.robots-txt]` selects, with `module`, the loading cache that holds each robots.txt module's answer, and sets the lifetimes of its copies, in seconds: `memory_seconds` (default 60), `store_seconds` (default 604800) and `store_idle_seconds` (default 2592000). A memory lifetime may be 0, the others must be at least 1, and none may be more than a year. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index e3fec94be1..175073a01a 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1038,7 +1038,7 @@ Every adapter removes the three from a response before it finalizes it, unless t ## robots.txt Configuration -What `/robots.txt` answers. With no `[robots-txt]` section the publisher keeps their own file and `/robots.txt` reaches the origin like any other path. +What `/robots.txt` answers. With a `[robots-txt]` section, and a `[[fetch]]` or `[[serve]]` entry for `text/plain` on the file's path naming `robots_txt`, core's own middleware makes the file, replacing the origin's own where it has one and making it where it has none. With no section the publisher keeps their own file and `/robots.txt` reaches the origin like any other path. ### `[robots-txt]` @@ -1060,10 +1060,11 @@ A module with settings of its own reads them from its own block, `[robots-txt. Date: Mon, 12 Oct 2026 15:18:27 +0100 Subject: [PATCH 03/23] Only a fetch entry on /robots.txt places the robots.txt middleware The settings refuse robots_txt named on any other path or media type and require a fetch entry on /robots.txt. An origin page answered at that path is replaced by the file, as a single-page application's catch-all would otherwise be served to crawlers. Axum again proves a refusal tags every response, robots.txt names resolve through the shared module name rule, the unused registry lookup is gone and the API reference drops the removed route. --- CHANGELOG.md | 2 +- .../tests/seam_probe.rs | 46 +++++++ .../src/integrations/registry.rs | 21 +--- crates/trusted-server-core/src/publisher.rs | 48 +++++++- .../src/robots_txt/middleware.rs | 12 +- .../trusted-server-core/src/robots_txt/mod.rs | 8 +- crates/trusted-server-core/src/settings.rs | 114 +++++++++++++++--- crates/trusted-server-modules/src/lib.rs | 5 +- docs/guide/api-reference.md | 25 ---- docs/guide/configuration.md | 6 +- 10 files changed, 207 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f55f586c93..43d306117c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,7 +81,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `robots_txt`, core's own middleware for the file `[robots-txt]` describes, named in a `[[fetch]]` or `[[serve]]` entry for `text/plain` on `/robots.txt`. Its preparation makes the file from the selected modules through their holding, and its action replaces whatever the origin sent with it, so the file replaces the origin's own and the creating rule makes it where the origin has none. A module that cannot answer with nothing held refuses the request with `503` and `Retry-After` before any document is processed. A section with no entry naming the middleware is refused when the settings load. +- `robots_txt`, core's own middleware for the file `[robots-txt]` describes, named in a `[[fetch]]` entry for `text/plain` on `/robots.txt`. Its preparation makes the file from the selected modules through their holding, and its action replaces the origin's file with it, and the creating rule makes it where the origin answers `404`, or `200` with a page. A module that cannot answer with nothing held refuses the request with `503` and `Retry-After` before any document is processed. A section with no such entry, or with an entry naming the middleware on another path or media type, is refused when the settings load. - A middleware may prepare, and may refuse. `Middleware::prepare` runs in the request's handler before the document is processed, as the middleware's module with the request's services, and what it answers is handed to `create` under the middleware's name, so a decision that needs a store or the network is made there while `create` stays synchronous. A refusal answers the request with its status and `Retry-After`, with no document, and nothing is stored. - A loading cache. `PlatformLoadingCache` in core loads a missing value once per key however many requests ask for it at the same moment, gives every waiting request the value or the error, and holds nothing that failed. A crate offers one through `IntegrationBuilder::with_loading_cache`, each feature that holds answers selects the module whose cache holds them in its own `[cache.]` table, and an adapter takes the selections with `loading_caches_from`. A feature that selects none gets `NoLoadingCache`, which loads every time. The new `crates/loading-cache/51degrees`, the module `loading-cache.51degrees`, implements it with the published `fiftyone-caching` package as a brief copy in process memory over the host's value cache or key-value store, and the stock list ships it, so every adapter takes it. Every load runs in its caller. - `[cache.robots-txt]` selects, with `module`, the loading cache that holds each robots.txt module's answer, and sets the lifetimes of its copies, in seconds: `memory_seconds` (default 60), `store_seconds` (default 604800) and `store_idle_seconds` (default 2592000). A memory lifetime may be 0, the others must be at least 1, and none may be more than a year. diff --git a/crates/trusted-server-adapter-axum/tests/seam_probe.rs b/crates/trusted-server-adapter-axum/tests/seam_probe.rs index 02c9c1a113..73d7310651 100644 --- a/crates/trusted-server-adapter-axum/tests/seam_probe.rs +++ b/crates/trusted-server-adapter-axum/tests/seam_probe.rs @@ -256,6 +256,52 @@ async fn proxy_route_reports_the_modules_geo_and_that_the_preparer_ran() { ); } +/// With `refuse_all` selected every response this adapter finalizes says not +/// to index, and the file itself is the publisher path's, made by the +/// `robots_txt` middleware the entry names, so no route of this server names +/// the path. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn refusing_every_crawler_tags_every_response_and_registers_no_route() { + let settings = || { + settings_with(&format!( + r#" + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + {PROBE_BLOCK} + "# + )) + }; + let registered: Vec = + TrustedServerApp::routes_with_registrations(settings(), &[seam_probe::builder()]) + .expect("should build the routes") + .routes() + .into_iter() + .map(|route| route.path().to_string()) + .collect(); + assert!( + !registered.iter().any(|path| path == "/robots.txt"), + "the file is the publisher path's: {registered:?}" + ); + + let mut service = service_with(settings(), &[seam_probe::builder()]); + let other = get(&mut service, seam_probe::SEAM_PROBE_REPORT_PATH).await; + + assert_eq!( + other + .headers() + .get("x-robots-tag") + .and_then(|value| value.to_str().ok()), + Some("noindex, nofollow"), + "no route may be the one page a crawler is allowed to index" + ); +} + /// A name `[robots-txt] modules` gives that nothing the deployment runs /// supplies stops the adapter at startup, so the file does not fail on the /// first crawler to ask for it. diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index cdfb63d107..bd571a16da 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1820,9 +1820,7 @@ impl IntegrationRegistry { inner.geo_module = geo_module; check_robots_txt_selection(settings, &inner)?; if let Some(config) = settings.robots_txt.as_ref() { - // Core's own middleware for the file, made here because the - // contributors it asks are known only once every registration - // is in, and a middleware cannot reach the registry. + // Core's own middleware for the file, made once every registration is in. let selected = crate::robots_txt::select(config, &inner.robots_txt_contributors)?; inner.middleware.push(( crate::robots_txt::middleware::MODULE, @@ -1870,23 +1868,6 @@ impl IntegrationRegistry { self.inner.device_module.clone() } - /// The robots.txt contributor `[robots-txt] modules` means by `written`, - /// with the name it was declared under, or `None` when nothing this - /// deployment runs supplies one under that name. - #[must_use] - pub fn robots_txt_contributor( - &self, - written: &str, - ) -> Option<(&'static str, Arc)> { - let offered = &self.inner.robots_txt_contributors; - let names: Vec<&str> = offered.iter().map(|(name, _)| *name).collect(); - let name = crate::module_name::resolve(ROBOTS_TXT_TYPE, written, &names)?; - offered - .iter() - .find(|(offered_name, _)| *offered_name == name) - .map(|(offered_name, contributor)| (*offered_name, Arc::clone(contributor))) - } - /// The name of every middleware a running module supplies, in /// registration order. #[must_use] diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6ef46be3cd..b91a92e507 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1974,6 +1974,26 @@ pub(crate) fn classify_response_route( ResponseRoute::Stream } +/// Whether the origin answered with a page where `entry` places core's +/// robots.txt middleware, as a single-page application's catch-all does. A +/// crawler reads a page there as no rules at all, so the file is made as it is +/// where the origin has none. +fn answers_robots_txt_with_a_page( + response: &Response, + entry: &crate::middleware::PhaseEntry, +) -> bool { + response.status() == StatusCode::OK + && entry + .middleware + .iter() + .any(|name| name == crate::robots_txt::middleware::MODULE) + && response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(is_html_content_type) +} + /// An empty document of `media_type`, which the middleware a `[[fetch]]` /// entry names for a path the origin has nothing at is run on. fn empty_document(media_type: &str) -> Result, Report> { @@ -5686,10 +5706,12 @@ pub async fn handle_publisher_request( response.headers().len() ); - // A file the origin does not have is made, see `PhaseEntries::making`. - if response.status() == StatusCode::NOT_FOUND - && (request_method == Method::GET || request_method == Method::HEAD) + // A file the origin does not have is made, see `PhaseEntries::making`. An + // origin page where the robots.txt middleware is placed is no file either. + if (request_method == Method::GET || request_method == Method::HEAD) && let Some(entry) = settings.fetch.making(&request_path) + && (response.status() == StatusCode::NOT_FOUND + || answers_robots_txt_with_a_page(&response, entry)) { log::debug!( "Making {} for {request_path}, which the origin does not have, with {}", @@ -21339,6 +21361,26 @@ mod tests { ); } + /// An origin that answers every path with its page, as a single-page + /// application's catch-all does, has no file, so the middleware makes it + /// rather than the crawler being handed the page. + #[tokio::test] + async fn robots_txt_replaces_a_page_the_origin_answers_at_its_path() { + let proxied = proxy_with( + settings_refusing_every_crawler(), + Method::GET, + crate::robots_txt::ROBOTS_TXT_PATH, + 200, + ) + .await; + + assert_eq!( + streamed_robots_txt(proxied).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the file replaces the page" + ); + } + /// Where the origin has no file, the creating rule makes it. #[tokio::test] async fn robots_txt_is_made_where_the_origin_has_none() { diff --git a/crates/trusted-server-core/src/robots_txt/middleware.rs b/crates/trusted-server-core/src/robots_txt/middleware.rs index a92427163a..6523c912bc 100644 --- a/crates/trusted-server-core/src/robots_txt/middleware.rs +++ b/crates/trusted-server-core/src/robots_txt/middleware.rs @@ -1,8 +1,8 @@ //! `/robots.txt` as a middleware of core's own, named `robots_txt` in a -//! `[[fetch]]` or `[[serve]]` entry on the file's path. Its preparation makes -//! the file from the selected contributors, through their holding, and its -//! action replaces whatever the origin sent with that file. Where the origin -//! has no file, the creating rule runs it on an empty one. +//! `[[fetch]]` entry on the file's path. Its preparation makes the file from +//! the selected contributors, through their holding, and its action replaces +//! the origin's file with it. Where the origin has no file, or answers with a +//! page, the creating rule runs it on an empty one. use std::any::Any; use std::time::Duration; @@ -108,9 +108,7 @@ impl Middleware for RobotsTxt { stream: Some(Box::new(ReplaceBody { file: Some(file) })), ..MiddlewareAction::pass() }, - // Nothing prepared means the preparation did not run, which no - // request can cause. The origin's own file is left as it is, - // because an empty file would tell crawlers to crawl everything. + // Nothing prepared means the preparation did not run, which no request can cause. None => { log::error!("robots.txt was not prepared, so the origin's file is served as it is"); MiddlewareAction::pass() diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs index e02b4c925d..09b58bec5e 100644 --- a/crates/trusted-server-core/src/robots_txt/mod.rs +++ b/crates/trusted-server-core/src/robots_txt/mod.rs @@ -304,9 +304,11 @@ pub(crate) fn select( selected.push((ALLOW_ALL, None)); continue; } - let Some((declared, contributor)) = contributors.iter().find(|(name, _)| { - *name == written || crate::module_name::short_form(MODULE_TYPE, name) == written - }) else { + let names: Vec<&str> = contributors.iter().map(|(name, _)| *name).collect(); + let Some((declared, contributor)) = + crate::module_name::resolve(MODULE_TYPE, written, &names) + .and_then(|declared| contributors.iter().find(|(name, _)| *name == declared)) + else { return Err(Report::new(configuration(format!( "[robots-txt] modules names `{written}`, which no module in this deployment supplies" )))); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 0e0bffb5c9..e6155b8ce4 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -4319,26 +4319,55 @@ impl Settings { Ok(()) } - /// Refuses a `[robots-txt]` section that no `[[fetch]]` or `[[serve]]` - /// entry for `text/plain` names the `robots_txt` middleware for, because - /// the file would never be made. + /// Refuses a `[robots-txt]` section unless a `[[fetch]]` entry for + /// `text/plain` with `path = "/robots.txt"` names the `robots_txt` + /// middleware, and refuses any entry naming it on another path or media + /// type. Only a `[[fetch]]` entry makes the file where the origin has + /// none, and an entry on another path would replace that file with the + /// robots.txt rules. fn require_robots_txt_placed(&self) -> Result<(), Report> { + use crate::robots_txt::ROBOTS_TXT_PATH; use crate::robots_txt::middleware::{MEDIA_TYPE, MODULE}; - let placed = [&self.fetch, &self.serve].iter().any(|entries| { - entries.entries().iter().any(|entry| { - entry.media_type == MEDIA_TYPE && entry.middleware.iter().any(|name| name == MODULE) - }) - }); - if placed { + let names = |entry: &crate::middleware::PhaseEntry| { + entry.middleware.iter().any(|name| name == MODULE) + }; + let on_the_file = |entry: &crate::middleware::PhaseEntry| { + entry.media_type == MEDIA_TYPE && entry.path.as_deref() == Some(ROBOTS_TXT_PATH) + }; + for (phase, entries) in [("fetch", &self.fetch), ("serve", &self.serve)] { + for (index, entry) in entries.entries().iter().enumerate() { + if names(entry) && !on_the_file(entry) { + return Err(Report::new(TrustedServerError::Settings { + message: format!( + "[[{phase}]] entry {} names `{MODULE}` for `{}` on {}, where it \ + would replace each file it covers with robots.txt. Name it only \ + in an entry with media_type = `{MEDIA_TYPE}` and path = \ + `{ROBOTS_TXT_PATH}`", + index + 1, + entry.media_type, + entry.path.as_deref().map_or_else( + || "every path".to_owned(), + |path| format!("`{path}`") + ), + ), + })); + } + } + } + if self + .fetch + .entries() + .iter() + .any(|entry| names(entry) && on_the_file(entry)) + { return Ok(()); } Err(Report::new(TrustedServerError::Settings { message: format!( - "[robots-txt] is set, but no [[fetch]] or [[serve]] entry for `{MEDIA_TYPE}` \ - names `{MODULE}`, so the file would never be made. Add an entry with \ - media_type = `{MEDIA_TYPE}`, path = `{}` and the middleware", - crate::robots_txt::ROBOTS_TXT_PATH + "[robots-txt] is set, but no [[fetch]] entry with media_type = `{MEDIA_TYPE}` and \ + path = `{ROBOTS_TXT_PATH}` names `{MODULE}`, so the file would never be made \ + where the origin has none. Add that entry" ), })) } @@ -5119,11 +5148,68 @@ mod tests { let message = format!("{error:?}"); assert!( - message.contains("no [[fetch]] or [[serve]] entry for `text/plain` names `robots_txt`"), + message.contains( + "no [[fetch]] entry with media_type = `text/plain` and path = `/robots.txt` names \ + `robots_txt`" + ), + "{message}" + ); + } + + /// The `[[fetch]]` or `[[serve]]` entry placing the middleware on + /// `/robots.txt`. + const ROBOTS_TXT_ENTRY: &str = + "media_type = \"text/plain\"\npath = \"/robots.txt\"\nmiddleware = [\"robots_txt\"]\n"; + + /// Settings with `[robots-txt]` refusing every crawler and `entries`. + fn settings_str_placing_robots_txt(entries: &str) -> String { + format!( + "{}\n[robots-txt]\nmodules = \"refuse_all\"\n\n{entries}\n", + crate_test_settings_str() + ) + } + + #[test] + fn a_fetch_entry_on_the_files_path_places_robots_txt() { + let toml = settings_str_placing_robots_txt(&format!( + "[[fetch]]\n{ROBOTS_TXT_ENTRY}\n[[serve]]\n{ROBOTS_TXT_ENTRY}" + )); + + Settings::from_toml(&toml).expect("an entry on the file's path places the middleware"); + } + + /// An entry with no path would replace every text file, `ads.txt` + /// included, with the robots.txt rules. + #[test] + fn an_entry_naming_robots_txt_on_another_path_is_refused() { + let toml = settings_str_placing_robots_txt(&format!( + "[[fetch]]\n{ROBOTS_TXT_ENTRY}\n[[fetch]]\nmedia_type = \"text/plain\"\nmiddleware = [\"robots_txt\"]\n" + )); + + let error = Settings::from_toml(&toml).expect_err("every text file would be robots.txt"); + + let message = format!("{error:?}"); + assert!( + message.contains("[[fetch]] entry 2 names `robots_txt` for `text/plain` on every path"), "{message}" ); } + /// Only a `[[fetch]]` entry makes the file where the origin has none, so + /// a `[[serve]]` placement alone would leave an origin's `404` standing, + /// which a crawler reads as permission to crawl everything. + #[test] + fn robots_txt_placed_only_in_serve_is_refused() { + let toml = settings_str_placing_robots_txt(&format!("[[serve]]\n{ROBOTS_TXT_ENTRY}")); + + let error = Settings::from_toml(&toml).expect_err("the file would never be made"); + + assert!( + format!("{error:?}").contains("no [[fetch]] entry with media_type = `text/plain`"), + "{error:?}" + ); + } + #[test] fn the_robots_txt_section_is_read_by_its_hyphenated_name() { let toml = settings_str_with_robots_txt( diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index dfbaadaecc..2ca88f856b 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -259,9 +259,6 @@ mod tests { checked.extend(running); } - // Core's robots.txt middleware is registered by the section, not by a - // module, and `core_s_robots_txt_middleware_is_registered_with_the_section` - // checks it. checked.push(trusted_server_core::robots_txt::middleware::MODULE); for row in &listed { assert!( @@ -286,7 +283,7 @@ modules = \"refuse_all\" [[fetch]] media_type = \"text/plain\" -\n path = \"/robots.txt\" +path = \"/robots.txt\" middleware = [\"robots_txt\"] ", ); diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 06de03810a..be53893ddf 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -56,7 +56,6 @@ authority, scheme, and client-address headers in the innermost middleware. | normal | `/first-party/proxy` | `GET` | literal | `always` | real | real | real | real | | normal | `/first-party/sign` | `GET`, `POST` | literal | `always` | real | real | real | real | | normal | `/health` | `GET` | literal | `always` | real | real | — | real | -| normal | `/robots.txt` | `GET`, `HEAD` | literal | `settings.robots_txt` | real | real | real | real | | normal | `/static/tsjs=` | `GET` | template | `path.starts_with(/static/tsjs=)` | real | real | real | real | | normal | `/verify-signature` | `POST` | literal | `always` | real | real | real | real | | normal | `/{*rest}` | `DELETE`, `GET`, `HEAD`, `OPTIONS`, `PATCH`, `POST`, `PUT` | template | `publisher_fallback` | publisher fallback | publisher fallback | publisher fallback | publisher fallback | @@ -1027,30 +1026,6 @@ credentialed or otherwise private responses are forced `private, no-store`. No dedicated CORS grant or rate limiter is installed. Example: if the prefix is `/assets/`, request `curl -I https://edge.example.com/assets/logo.svg`. -### GET /robots.txt - -The publisher's `robots.txt`, written from the rules the modules named by -`[robots-txt] modules` give, in that order, as -[Configuration](/guide/configuration#robots-txt-configuration) describes. - -**Contract:** Auth: none. Request body: not applicable. Rate limit: none. -Endpoint-specific CORS: none. Answered only when the settings carry -`[robots-txt]`. Without the section the path reaches the publisher's origin, so -a publisher keeps their own file. `HEAD` is answered with the headers of the -`GET` and no body. Other methods reach the publisher's origin. - -**Response:** - -- `200 OK` with the file as `text/plain; charset=utf-8` and - `Cache-Control: public, max-age=3600` -- `503 Service Unavailable` with `Retry-After: 600` and - `Cache-Control: no-store` when a selected module cannot answer and no earlier - answer of its is held, because an empty file or a `404` reads to a crawler as - permission to crawl everything - -With `refuse_all` selected the file refuses every crawler, and every response -the deployment finalizes also carries `X-Robots-Tag: noindex, nofollow`. - ### Publisher fallback: `/` and `/{*rest}` **Contract:** `GET`, `POST`, `HEAD`, `OPTIONS`, `PUT`, `PATCH`, and `DELETE` diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 9e36bad46a..0f73b8dc11 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1038,7 +1038,7 @@ Every adapter removes the three from a response before it finalizes it, unless t ## robots.txt Configuration -What `/robots.txt` answers. With a `[robots-txt]` section, and a `[[fetch]]` or `[[serve]]` entry for `text/plain` on the file's path naming `robots_txt`, core's own middleware makes the file, replacing the origin's own where it has one and making it where it has none. With no section the publisher keeps their own file and `/robots.txt` reaches the origin like any other path. +What `/robots.txt` answers. With a `[robots-txt]` section, and a `[[fetch]]` entry for `text/plain` on the file's path naming `robots_txt`, core's own middleware makes the file, replacing the origin's own where it has one and making it where it has none. With no section the publisher keeps their own file and `/robots.txt` reaches the origin like any other path. ### `[robots-txt]` @@ -1060,11 +1060,11 @@ A module with settings of its own reads them from its own block, `[robots-txt. Date: Mon, 12 Oct 2026 23:05:07 +0100 Subject: [PATCH 04/23] The robots.txt middleware takes what it prepared for its own phase The middleware reads its prepared file with MiddlewareContext::take_prepared, and its tests keep and read what they prepare under the phase they run in, as the preparation now keys what it keeps by phase. --- crates/trusted-server-core/src/robots_txt/middleware.rs | 5 +++-- crates/trusted-server-core/src/robots_txt/mod.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/robots_txt/middleware.rs b/crates/trusted-server-core/src/robots_txt/middleware.rs index 6523c912bc..a51bc37c5c 100644 --- a/crates/trusted-server-core/src/robots_txt/middleware.rs +++ b/crates/trusted-server-core/src/robots_txt/middleware.rs @@ -103,7 +103,7 @@ impl Middleware for RobotsTxt { } fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { - match context.prepared.take::(MODULE) { + match context.take_prepared::(MODULE) { Some(file) => MiddlewareAction { stream: Some(Box::new(ReplaceBody { file: Some(file) })), ..MiddlewareAction::pass() @@ -171,6 +171,7 @@ mod tests { let state = IntegrationDocumentState::default(); let prepared = Prepared::default(); prepared.insert( + MiddlewarePhase::Fetch, MODULE, Box::new(String::from("User-agent: *\nDisallow: /\n")), ); @@ -208,7 +209,7 @@ mod tests { fn a_value_of_another_type_under_the_name_is_not_a_file() { let state = IntegrationDocumentState::default(); let prepared = Prepared::default(); - prepared.insert(MODULE, Box::new(7_u32)); + prepared.insert(MiddlewarePhase::Serve, MODULE, Box::new(7_u32)); let context = context_with(MiddlewarePhase::Serve, &state, &prepared); assert!(middleware().create(&context).stream.is_none()); diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs index 09b58bec5e..ea6f030794 100644 --- a/crates/trusted-server-core/src/robots_txt/mod.rs +++ b/crates/trusted-server-core/src/robots_txt/mod.rs @@ -577,7 +577,7 @@ mod tests { ..crate::middleware::test_support::context(MiddlewarePhase::Fetch, &state) }; chain.prepare(&module_context, &context).await?; - Ok(prepared.take::(middleware::MODULE)) + Ok(prepared.take::(MiddlewarePhase::Fetch, middleware::MODULE)) } /// `settings` with `config` as its `[robots-txt]` section and the entry From 7f7b58a73fa97c86600c7af011afacb607f7198a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 11:34:58 +0100 Subject: [PATCH 05/23] With nothing prepared the robots.txt file refuses every crawler The middleware's action passed the origin's file on when its preparation had left nothing, so a fault that skipped the preparation would serve whatever the origin holds, or no file at all, which a crawler reads as permission to crawl everything. The action now writes the refusal, with the `always_allow` paths kept open, whatever `modules` selects. No request reaches the action without its preparation today, because core prepares every response it streams, so this is the answer to a fault and not a path in use. The tests write the file with nothing prepared under an `allow_all` selection, and with a value of another type under the middleware's name. --- .../src/robots_txt/middleware.rs | 62 +++++++++++++------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/crates/trusted-server-core/src/robots_txt/middleware.rs b/crates/trusted-server-core/src/robots_txt/middleware.rs index a51bc37c5c..5929c3a2a0 100644 --- a/crates/trusted-server-core/src/robots_txt/middleware.rs +++ b/crates/trusted-server-core/src/robots_txt/middleware.rs @@ -103,16 +103,16 @@ impl Middleware for RobotsTxt { } fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { - match context.take_prepared::(MODULE) { - Some(file) => MiddlewareAction { - stream: Some(Box::new(ReplaceBody { file: Some(file) })), - ..MiddlewareAction::pass() - }, - // Nothing prepared means the preparation did not run, which no request can cause. - None => { - log::error!("robots.txt was not prepared, so the origin's file is served as it is"); - MiddlewareAction::pass() - } + let file = context.take_prepared::(MODULE).unwrap_or_else(|| { + // Nothing prepared means the preparation did not run. The file + // then refuses every crawler, because the origin's file, or no + // file, could allow what the selected rules refuse. + log::error!("robots.txt was not prepared, so the file refuses every crawler"); + document::refusal(&self.config.always_allow) + }); + MiddlewareAction { + stream: Some(Box::new(ReplaceBody { file: Some(file) })), + ..MiddlewareAction::pass() } } } @@ -193,26 +193,48 @@ mod tests { assert_eq!(output, b"User-agent: *\nDisallow: /\n"); } - #[test] - fn with_nothing_prepared_the_origins_file_is_left_as_it_is() { + /// What `middleware` writes in place of an origin's file in `phase`, with + /// `prepared` as what its preparation left. + fn written(middleware: &RobotsTxt, phase: MiddlewarePhase, prepared: &Prepared) -> String { let state = IntegrationDocumentState::default(); - let prepared = Prepared::default(); - let context = context_with(MiddlewarePhase::Fetch, &state, &prepared); + let mut processor = middleware + .create(&context_with(phase, &state, prepared)) + .stream + .expect("the middleware should write a file whatever was prepared"); + let output = processor + .process_chunk(b"User-agent: *\nAllow: /\n", true) + .expect("should write the file"); + String::from_utf8(output).expect("the file is text") + } - let action = middleware().create(&context); + /// Nothing prepared means the preparation did not run, so the file + /// refuses every crawler, whatever `modules` selects, rather than letting + /// the origin's file through. + #[test] + fn with_nothing_prepared_the_file_refuses_every_crawler() { + let allowing: RobotsTxtConfig = serde_json::from_value(serde_json::json!({ + "modules": "allow_all", + "always_allow": ["/ads.txt"], + })) + .expect("should read the robots.txt settings"); + let middleware = RobotsTxt::new(allowing, vec![(crate::robots_txt::ALLOW_ALL, None)]); - assert!(action.stream.is_none(), "should pass the body through"); - assert!(action.head_inserts.is_empty()); + assert_eq!( + written(&middleware, MiddlewarePhase::Fetch, &Prepared::default()), + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n" + ); } #[test] fn a_value_of_another_type_under_the_name_is_not_a_file() { - let state = IntegrationDocumentState::default(); let prepared = Prepared::default(); prepared.insert(MiddlewarePhase::Serve, MODULE, Box::new(7_u32)); - let context = context_with(MiddlewarePhase::Serve, &state, &prepared); - assert!(middleware().create(&context).stream.is_none()); + assert_eq!( + written(&middleware(), MiddlewarePhase::Serve, &prepared), + "User-agent: *\nDisallow: /\n", + "the file refuses every crawler, as with nothing prepared" + ); } #[test] From 146b43837cc1ea8c2e6486100d0cd1a415f55b98 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 11:42:33 +0100 Subject: [PATCH 06/23] The robots.txt file takes the place of any answer it cannot replace With `refuse_all` selected the origin's answer still reached a crawler wherever the middleware was not handed it as a `200` of `text/plain`. A `401`, `403` or `410` reads to a crawler as permission to crawl everything, a redirect sends it to rules of the origin's choosing, and a file in another charset, in an encoding core cannot read or of another type kept the origin's rules, so the refusal could be loosened. A middleware now says with `writes_whole_document` that it writes the whole document whatever the origin sent. Where a `[[fetch]]` entry with a `path` names one, core makes the document on an empty one in place of any answer it would not hand the middleware as a `200` of the entry's media type, as it already did for a `404`. The robots.txt middleware says so, and the neutral rule replaces the rule for a page at the file's path, which named the middleware in core. A request that names no host is left as it was, because core runs no middleware for it. The module comment and the comment on `refuse_all` say that the origin is still asked and that the file takes the place of whatever it answers. The test sends eleven answers, among them `401`, `403`, `301`, `204`, `206` and files core cannot replace, and each becomes the refusal with status `200`. --- crates/trusted-server-core/src/middleware.rs | 19 ++ crates/trusted-server-core/src/publisher.rs | 219 +++++++++++++++--- .../src/robots_txt/middleware.rs | 12 +- .../trusted-server-core/src/robots_txt/mod.rs | 7 +- 4 files changed, 222 insertions(+), 35 deletions(-) diff --git a/crates/trusted-server-core/src/middleware.rs b/crates/trusted-server-core/src/middleware.rs index 35fe3544f4..032ee86183 100644 --- a/crates/trusted-server-core/src/middleware.rs +++ b/crates/trusted-server-core/src/middleware.rs @@ -489,6 +489,16 @@ pub trait Middleware: Send + Sync { media_type == HTML_MEDIA_TYPE } + /// Whether this middleware writes the whole document, whatever the origin + /// sent. Where a `[[fetch]]` entry with a `path` names it, core makes the + /// document on an empty one in place of any answer it would not hand the + /// middleware as a `200` of the entry's media type, as it does where the + /// origin answers `404`. `false` by default, for a middleware that changes + /// what the origin sent. + fn writes_whole_document(&self) -> bool { + false + } + /// The phases an entry may name this middleware in. fn phases(&self) -> &[MiddlewarePhase]; @@ -797,6 +807,15 @@ impl MiddlewareChain { self.middleware.is_empty() } + /// Whether a middleware in the chain writes the whole document, see + /// [`Middleware::writes_whole_document`]. + #[must_use] + pub fn writes_whole_document(&self) -> bool { + self.middleware + .iter() + .any(|middleware| middleware.writes_whole_document()) + } + /// Runs the preparation of every middleware that has one, in the order /// they run, keeping what each prepared in the context's [`Prepared`]. /// Each is called as the module it belongs to, with the permissions it diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index e40f360de2..7179aa5fad 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -2045,24 +2045,69 @@ pub(crate) fn classify_response_route( ResponseRoute::Stream } -/// Whether the origin answered with a page where `entry` places core's -/// robots.txt middleware, as a single-page application's catch-all does. A -/// crawler reads a page there as no rules at all, so the file is made as it is -/// where the origin has none. -fn answers_robots_txt_with_a_page( +/// The `[[fetch]]` entry whose middleware make the document at +/// `request_path` on an empty one, in place of the origin's `response`, see +/// [`PhaseEntries::making`](crate::middleware::PhaseEntries::making). That is +/// where the origin answers `404`, and where a middleware the entry names +/// writes the whole document and would not be handed the answer as a `200` +/// of the entry's media type, so that no other answer of the origin's is +/// served in place of the document. +fn entry_making_the_document<'s>( + settings: &'s Settings, + integration_registry: &IntegrationRegistry, + request_path: &str, + request_host: &str, + response: &Response, +) -> Option<&'s crate::middleware::PhaseEntry> { + let entry = settings.fetch.making(request_path)?; + if response.status() == StatusCode::NOT_FOUND { + return Some(entry); + } + let writes_whole_document = integration_registry + .middleware_chain( + &settings.fetch, + MiddlewarePhase::Fetch, + &entry.media_type, + request_path, + ) + .writes_whole_document(); + // Core runs no middleware for a request that names no host, so a document + // made for one would go out empty. + (writes_whole_document + && !request_host.is_empty() + && !is_handed_over_whole(response, &entry.media_type, request_host)) + .then_some(entry) +} + +/// Whether core hands a middleware for `media_type` the origin's `response` +/// whole, being a `200` of that media type, in a charset and an encoding +/// core reads, that core streams through the middleware. +fn is_handed_over_whole( response: &Response, - entry: &crate::middleware::PhaseEntry, + media_type: &str, + request_host: &str, ) -> bool { + let content_type = response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + let content_encoding = response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); response.status() == StatusCode::OK - && entry - .middleware - .iter() - .any(|name| name == crate::robots_txt::middleware::MODULE) - && response - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .is_some_and(is_html_content_type) + && media_type_of(content_type) == media_type + && crate::middleware::is_utf8_or_undeclared_charset(content_type) + && classify_response_route( + response.status(), + content_type, + &content_encoding, + request_host, + ) == ResponseRoute::Stream } /// An empty document of `media_type`, which the middleware a `[[fetch]]` @@ -5804,12 +5849,15 @@ pub async fn handle_publisher_request( response.headers().len() ); - // A file the origin does not have is made, see `PhaseEntries::making`. An - // origin page where the robots.txt middleware is placed is no file either. + // A file the origin does not have is made, see `entry_making_the_document`. if (request_method == Method::GET || request_method == Method::HEAD) - && let Some(entry) = settings.fetch.making(&request_path) - && (response.status() == StatusCode::NOT_FOUND - || answers_robots_txt_with_a_page(&response, entry)) + && let Some(entry) = entry_making_the_document( + settings, + integration_registry, + &request_path, + request_host, + &response, + ) { log::debug!( "Making {} for {request_path}, which the origin does not have, with {}", @@ -21723,13 +21771,46 @@ mod tests { body: &[u8], content_type: &'static str, ) -> Proxied { - let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + proxy_answered( + settings, + publisher_request(method, path, &[]), status, - body.to_vec(), + body, vec![("content-type", content_type)], - ); + ) + .await + } + + /// A request for `path` on the publisher's host, carrying `headers`. + fn publisher_request( + method: Method, + path: &str, + headers: &[(&'static str, &'static str)], + ) -> HttpRequest { + let mut builder = HttpRequest::builder() + .method(method) + .uri(format!("https://publisher.example{path}")) + .header(header::HOST, "publisher.example"); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + builder + .body(EdgeBody::empty()) + .expect("should build request") + } + + /// `req` through the publisher path, the origin answering `status` with + /// `body` and `headers`. + async fn proxy_answered( + settings: Settings, + req: HttpRequest, + status: u16, + body: &[u8], + headers: Vec<(&'static str, &'static str)>, + ) -> Proxied { + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers(status, body.to_vec(), headers); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -21741,12 +21822,6 @@ mod tests { let orchestrator = AuctionOrchestrator::new( crate::auction::test_support::legacy_auction_config(&settings), ); - let req = HttpRequest::builder() - .method(method) - .uri(format!("https://publisher.example{path}")) - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); let response = handle_publisher_request( AppContext { settings: &settings, @@ -21912,6 +21987,88 @@ mod tests { ); } + /// With `refuse_all` selected no other answer of the origin's takes the + /// place of the file. A crawler takes an error, a redirect or an empty + /// answer as permission, and a file core cannot replace as the origin's + /// rules, so the refusal is made in place of each, with status `200`. + #[tokio::test] + async fn robots_txt_is_made_in_place_of_any_answer_the_middleware_cannot_replace() { + let permissive: &[u8] = b"User-agent: *\nAllow: /\n"; + let plain = ("content-type", "text/plain"); + let answers: [(u16, &[u8], Vec<(&'static str, &'static str)>); 11] = [ + (401, b"login", vec![("content-type", "text/html")]), + (403, permissive, vec![plain]), + (410, b"", vec![plain]), + ( + 301, + b"", + vec![("location", "https://elsewhere.example/robots.txt")], + ), + (302, b"", vec![("location", "/robots-staging.txt")]), + (500, b"down", vec![("content-type", "text/html")]), + (204, b"", Vec::new()), + ( + 200, + permissive, + vec![("content-type", "text/plain; charset=iso-8859-1")], + ), + (200, permissive, vec![plain, ("content-encoding", "zstd")]), + ( + 200, + permissive, + vec![("content-type", "application/octet-stream")], + ), + ( + 206, + b"User-agent: *\n", + vec![plain, ("content-range", "bytes 0-13/23")], + ), + ]; + + for (status, body, headers) in answers { + let answer = format!("{status} {headers:?}"); + let proxied = proxy_answered( + settings_refusing_every_crawler(), + publisher_request(Method::GET, crate::robots_txt::ROBOTS_TXT_PATH, &[]), + status, + body, + headers, + ) + .await; + + assert_eq!( + streamed_robots_txt(proxied).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the refusal should take the place of {answer}" + ); + } + } + + /// Core runs no middleware for a request that names no host, so the + /// origin's answer stands there rather than an empty file being made. + #[tokio::test] + async fn robots_txt_for_a_request_that_names_no_host_keeps_the_origins_answer() { + let req = HttpRequest::builder() + .method(Method::GET) + .uri(crate::robots_txt::ROBOTS_TXT_PATH) + .body(EdgeBody::empty()) + .expect("should build request"); + + let proxied = proxy_answered( + settings_refusing_every_crawler(), + req, + 403, + b"forbidden", + vec![("content-type", "text/plain")], + ) + .await; + + assert_eq!( + response_of(proxied.response).status(), + StatusCode::FORBIDDEN + ); + } + #[tokio::test] async fn a_file_the_origin_does_not_have_is_made_by_the_middleware_named_for_its_path() { use crate::integrations::registry_test_support::middleware_fixture as fixture; diff --git a/crates/trusted-server-core/src/robots_txt/middleware.rs b/crates/trusted-server-core/src/robots_txt/middleware.rs index 5929c3a2a0..aa4034d48b 100644 --- a/crates/trusted-server-core/src/robots_txt/middleware.rs +++ b/crates/trusted-server-core/src/robots_txt/middleware.rs @@ -1,8 +1,9 @@ //! `/robots.txt` as a middleware of core's own, named `robots_txt` in a //! `[[fetch]]` entry on the file's path. Its preparation makes the file from //! the selected contributors, through their holding, and its action replaces -//! the origin's file with it. Where the origin has no file, or answers with a -//! page, the creating rule runs it on an empty one. +//! the origin's file with it. Where the origin answers with anything else, +//! being no file, a page, an error or a redirect among them, the creating rule +//! runs it on an empty one. use std::any::Any; use std::time::Duration; @@ -73,6 +74,13 @@ impl Middleware for RobotsTxt { media_type == MEDIA_TYPE } + // A crawler takes an error, a redirect, an empty answer, or an origin + // file the action cannot replace, as permission or as the origin's own + // rules, so the file is made in place of each. + fn writes_whole_document(&self) -> bool { + true + } + fn phases(&self) -> &[MiddlewarePhase] { &MiddlewarePhase::ALL } diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs index ea6f030794..9dbfb4affa 100644 --- a/crates/trusted-server-core/src/robots_txt/mod.rs +++ b/crates/trusted-server-core/src/robots_txt/mod.rs @@ -14,7 +14,9 @@ //! contributor says. //! //! 1. A refusal cannot be loosened. With `refuse_all` selected the file is the -//! refusal and nothing else. +//! refusal and nothing else, and the origin cannot loosen it either, +//! because the file takes the place of whatever the origin answers to a +//! `GET` or `HEAD` that names its host. //! 2. A path in `always_allow` stays open to every crawler a contribution //! refuses the whole site to, `*` included. A narrower rule is kept as //! given, so `Disallow: /ads` still closes off `/ads.txt`. @@ -159,7 +161,8 @@ impl RobotsTxtConfig { /// The configuration that refuses every crawler with no exception. /// /// The starting point for a publisher who wants the refusal with - /// allowances of their own. Needs no contributor, no key and no network. + /// allowances of their own. Needs no contributor and no key. The origin is + /// still asked, and the file takes the place of whatever it answers. #[must_use] pub fn refuse_all() -> Self { Self { From 63f076c4515d367a13e8b44312ac60b9017837e8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 11:44:49 +0100 Subject: [PATCH 07/23] A robots.txt contributor's failure is logged with each address cut to its host What a contributor reported when it failed was kept whole and written to the log, by the holding when it serves a kept answer past its age and by the middleware when it refuses the request, so an address the contributor named with a licence key in its path, query or user part would have put the key in the log. The text now keeps only the scheme and host of each address, and drops the rest of it, before the load error is built from it. The body of the `503` was and is a constant, so nothing of the text reaches a crawler. The test fails a contributor with an address carrying a secret in its user part, path, query and fragment, and a second address in quotes, and reads the error the middleware logs. --- .../src/robots_txt/holding.rs | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/robots_txt/holding.rs b/crates/trusted-server-core/src/robots_txt/holding.rs index 136b162adf..6f6dc20bf4 100644 --- a/crates/trusted-server-core/src/robots_txt/holding.rs +++ b/crates/trusted-server-core/src/robots_txt/holding.rs @@ -179,7 +179,9 @@ fn load(ask: Ask, kept: StoreKey) -> Loader { let contribution = contributor .contribute(context.call(id, contributor.required_permissions())) .await - .map_err(|report| LoadError::new(format!("{report:?}")))?; + .map_err(|report| { + LoadError::new(addresses_cut_to_their_host(&format!("{report:?}"))) + })?; let bytes = serde_json::to_vec(&contribution) .map_err(|error| LoadError::new(format!("the answer cannot be held: {error}")))?; keep_answer(&services, &kept, &bytes).await; @@ -188,6 +190,34 @@ fn load(ask: Ask, kept: StoreKey) -> Loader { }) } +/// `text` with every address in it cut to its scheme and host, because what +/// a contributor reports is written to the log, and the path, query or user +/// part of an address it names may carry a key. +fn addresses_cut_to_their_host(text: &str) -> String { + let mut cut = String::with_capacity(text.len()); + let mut rest = text; + while let Some(at) = rest.find("://") { + let (before, address) = rest.split_at(at + "://".len()); + cut.push_str(before); + // An address ends where white space or a quote does, so whatever + // else follows the host is dropped with it. + let end = address + .find(|c: char| c.is_whitespace() || matches!(c, '"' | '\'' | '<' | '>' | '`' | '\\')) + .unwrap_or(address.len()); + let authority = address[..end] + .split(['/', '?', '#']) + .next() + .unwrap_or_default(); + let host = authority + .rsplit_once('@') + .map_or(authority, |(_, host)| host); + cut.push_str(host); + rest = &address[end..]; + } + cut.push_str(rest); + cut +} + /// The key one contributor's answer for one fingerprint is held under, a hash /// so the fingerprint's contents, whatever they are, never appear in a store. fn held_key(id: &str, fingerprint: &str) -> String { @@ -759,6 +789,38 @@ pub(crate) mod tests { assert_eq!(scripted.times_asked(), 2); } + /// What a contributor reports when it fails is written to the log, so an + /// address it names keeps only its scheme and host, and a key in the + /// path, the query or the user part is not written. + #[tokio::test] + async fn a_failure_gives_only_the_host_of_an_address_it_names() { + let scripted = Arc::new(Scripted::new(vec![Err( + "GET https://user:secret-password@rules.example:8443/v1/secret-in-path/rules\ + ?licence=secret-in-query#secret-in-fragment failed, and \"http://other.example/\ + secret-in-quotes\" too" + .to_owned(), + )])); + let services = services_with(Arc::default(), Arc::default()); + + let error = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect_err("with nothing kept the failure is an error"); + let logged = format!("{error:?}"); + + assert!( + logged.contains("GET https://rules.example:8443 failed, and"), + "{logged}" + ); + assert!(logged.contains("http://other.example"), "{logged}"); + assert!(!logged.contains("secret"), "{logged}"); + } + /// A held copy that no longer reads is removed and the contributor asked /// again, rather than every request failing until the copy ends. #[tokio::test] From 016c07d6c4d7615a3dedb315474e0ffe94eb2628 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 11:47:59 +0100 Subject: [PATCH 08/23] The robots.txt middleware works on /robots.txt alone An entry's `path` is a prefix, so the entry with `path = "/robots.txt"` also covered `/robots.txt.bak`, `/robots.txtx` and `/robots.txt/rules`. On each of them a `text/plain` answer was replaced by the robots.txt file and a `404` was made into the file with status `200`, where the route this change removes answered `/robots.txt` alone. A middleware now says with `handles_path` which request paths it works on, every path by default, and the registry leaves it out of the chain for any other path the entry covers. Core makes no document where no middleware the entry names works on the path. The robots.txt middleware works on `/robots.txt` alone, where RFC 9309 places the file. The test sends a `200`, a `404` and a `403` for three longer paths, each of which keeps the origin's answer, and a `404` for the file's path with a query, which is made into the file. --- .../src/integrations/registry.rs | 6 +- crates/trusted-server-core/src/middleware.rs | 9 ++ crates/trusted-server-core/src/publisher.rs | 114 ++++++++++++++---- .../src/robots_txt/middleware.rs | 8 +- 4 files changed, 109 insertions(+), 28 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 5711b6ec8c..d95cfb1ade 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1920,8 +1920,9 @@ impl IntegrationRegistry { } /// The middleware the first entry covering a response selects, in the - /// order they run, for a response of `media_type` to a request for `path`. - /// A response no entry covers gives an empty chain. + /// order they run, for a response of `media_type` to a request for `path`, + /// leaving out any that does not work on `path`. A response no entry + /// covers gives an empty chain. #[must_use] pub fn middleware_chain( &self, @@ -1934,6 +1935,7 @@ impl IntegrationRegistry { .for_response(media_type, path) .iter() .filter_map(|name| self.inner.middleware_named(name).map(Arc::clone)) + .filter(|middleware| middleware.handles_path(path)) .collect(); MiddlewareChain::new(phase, media_type, middleware) } diff --git a/crates/trusted-server-core/src/middleware.rs b/crates/trusted-server-core/src/middleware.rs index 032ee86183..b67b033e99 100644 --- a/crates/trusted-server-core/src/middleware.rs +++ b/crates/trusted-server-core/src/middleware.rs @@ -489,6 +489,15 @@ pub trait Middleware: Send + Sync { media_type == HTML_MEDIA_TYPE } + /// Whether this middleware works on a response to a request for `path`, + /// every path by default. An entry's `path` is a prefix, so a middleware + /// made for one file says which path is that file, and is left out of + /// the chain for any other path the entry covers. + fn handles_path(&self, path: &str) -> bool { + let _ = path; + true + } + /// Whether this middleware writes the whole document, whatever the origin /// sent. Where a `[[fetch]]` entry with a `path` names it, core makes the /// document on an empty one in place of any answer it would not hand the diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 7179aa5fad..4f0a2225e1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -2048,10 +2048,10 @@ pub(crate) fn classify_response_route( /// The `[[fetch]]` entry whose middleware make the document at /// `request_path` on an empty one, in place of the origin's `response`, see /// [`PhaseEntries::making`](crate::middleware::PhaseEntries::making). That is -/// where the origin answers `404`, and where a middleware the entry names -/// writes the whole document and would not be handed the answer as a `200` -/// of the entry's media type, so that no other answer of the origin's is -/// served in place of the document. +/// where a middleware the entry names works on the path and the origin +/// answers `404`, and where such a middleware writes the whole document and +/// would not be handed the answer as a `200` of the entry's media type, so +/// that no other answer of the origin's is served in place of the document. fn entry_making_the_document<'s>( settings: &'s Settings, integration_registry: &IntegrationRegistry, @@ -2060,20 +2060,21 @@ fn entry_making_the_document<'s>( response: &Response, ) -> Option<&'s crate::middleware::PhaseEntry> { let entry = settings.fetch.making(request_path)?; + let chain = integration_registry.middleware_chain( + &settings.fetch, + MiddlewarePhase::Fetch, + &entry.media_type, + request_path, + ); + if chain.is_empty() { + return None; + } if response.status() == StatusCode::NOT_FOUND { return Some(entry); } - let writes_whole_document = integration_registry - .middleware_chain( - &settings.fetch, - MiddlewarePhase::Fetch, - &entry.media_type, - request_path, - ) - .writes_whole_document(); // Core runs no middleware for a request that names no host, so a document // made for one would go out empty. - (writes_whole_document + (chain.writes_whole_document() && !request_host.is_empty() && !is_handed_over_whole(response, &entry.media_type, request_host)) .then_some(entry) @@ -21896,23 +21897,16 @@ mod tests { settings } - /// The file a proxied robots.txt request streams, with its status and - /// content type checked. - async fn streamed_robots_txt(proxied: Proxied) -> String { + /// The response a proxied request streams, and the body streamed. + async fn streamed(proxied: Proxied) -> (Response, String) { let PublisherResponse::Stream { response, body, params, } = proxied.response else { - panic!("the file should stream through the entry's middleware"); + panic!("the answer should stream through the entries' middleware"); }; - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - content_type_of(&response), - Some("text/plain; charset=utf-8"), - "should be a document of the entry's media type" - ); let mut params = *params; let mut output = Vec::new(); stream_publisher_body_async( @@ -21925,8 +21919,24 @@ mod tests { &proxied.services, ) .await - .expect("should stream the file"); - String::from_utf8(output).expect("the file is text") + .expect("should stream the body"); + ( + response, + String::from_utf8(output).expect("the body is text"), + ) + } + + /// The file a proxied robots.txt request streams, with its status and + /// content type checked. + async fn streamed_robots_txt(proxied: Proxied) -> String { + let (response, file) = streamed(proxied).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + content_type_of(&response), + Some("text/plain; charset=utf-8"), + "should be a document of the entry's media type" + ); + file } /// The file the middleware makes replaces the origin's own. @@ -22069,6 +22079,60 @@ mod tests { ); } + /// An entry's `path` is a prefix, and the robots.txt middleware works on + /// the file's own path alone, so a longer path the entry covers keeps the + /// origin's answer, as it did when a route answered the file. A query is + /// no part of the path. + #[tokio::test] + async fn a_longer_path_the_robots_txt_entry_covers_keeps_the_origins_answer() { + for path in ["/robots.txt.bak", "/robots.txtx", "/robots.txt/rules"] { + let (response, body) = streamed( + proxy_origin( + settings_refusing_every_crawler(), + Method::GET, + path, + 200, + b"the origin's own text\n", + "text/plain; charset=utf-8", + ) + .await, + ) + .await; + assert_eq!(response.status(), StatusCode::OK, "{path}"); + assert_eq!(body, "the origin's own text\n", "{path} keeps its text"); + + for status in [404, 403] { + let proxied = proxy_origin( + settings_refusing_every_crawler(), + Method::GET, + path, + status, + b"", + "text/plain", + ) + .await; + assert_eq!( + response_of(proxied.response).status().as_u16(), + status, + "{path} keeps the origin's {status}" + ); + } + } + + let with_a_query = proxy_with( + settings_refusing_every_crawler(), + Method::GET, + "/robots.txt?crawler=one", + 404, + ) + .await; + assert_eq!( + streamed_robots_txt(with_a_query).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the file's path with a query is the file" + ); + } + #[tokio::test] async fn a_file_the_origin_does_not_have_is_made_by_the_middleware_named_for_its_path() { use crate::integrations::registry_test_support::middleware_fixture as fixture; diff --git a/crates/trusted-server-core/src/robots_txt/middleware.rs b/crates/trusted-server-core/src/robots_txt/middleware.rs index aa4034d48b..2efbd45eac 100644 --- a/crates/trusted-server-core/src/robots_txt/middleware.rs +++ b/crates/trusted-server-core/src/robots_txt/middleware.rs @@ -10,7 +10,7 @@ use std::time::Duration; use error_stack::Report; -use super::{RETRY_AFTER_SECONDS, RobotsTxtConfig, Selected, document, file_for}; +use super::{RETRY_AFTER_SECONDS, ROBOTS_TXT_PATH, RobotsTxtConfig, Selected, document, file_for}; use crate::error::TrustedServerError; use crate::middleware::{ Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, Preparation, Refusal, @@ -74,6 +74,12 @@ impl Middleware for RobotsTxt { media_type == MEDIA_TYPE } + // The file is the one path, as RFC 9309 section 2.3 gives it, so a longer + // path the entry's prefix covers keeps the origin's answer. + fn handles_path(&self, path: &str) -> bool { + path == ROBOTS_TXT_PATH + } + // A crawler takes an error, a redirect, an empty answer, or an origin // file the action cannot replace, as permission or as the origin's own // rules, so the file is made in place of each. From 56cf9bd98edb1649290b5bb55cb076415443fe1a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 11:51:05 +0100 Subject: [PATCH 09/23] The origin is asked for the whole file a middleware needs Core removes a reader's conditional and range headers before it asks the origin only for a page the ad stack answers and for a personalized HTML document, so a request for `/robots.txt` carried a crawler's `If-None-Match`, `If-Modified-Since`, `Range` and `If-Range` to the origin. The origin could then answer `304` for a copy the crawler holds, which may be the origin's own rules, or `206` with part of its file. A middleware now says with `needs_whole_body` that it needs the origin's whole body, and where an entry of either phase names one for the request's path, core removes the four headers as it does for those pages. The robots.txt middleware says so. The test sends the four headers with a request for the file, which reaches the origin without them, and with requests for two paths no such middleware works on, which keep them. --- .../src/integrations/registry.rs | 18 +++++ crates/trusted-server-core/src/middleware.rs | 24 +++++-- crates/trusted-server-core/src/publisher.rs | 69 +++++++++++++++++-- .../src/robots_txt/middleware.rs | 6 ++ 4 files changed, 108 insertions(+), 9 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index d95cfb1ade..4d5e17c08d 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1940,6 +1940,24 @@ impl IntegrationRegistry { MiddlewareChain::new(phase, media_type, middleware) } + /// Whether a middleware that an entry of either phase names for a request + /// for `path`, under any media type, needs the origin's whole body, see + /// [`Middleware::needs_whole_body`]. Asked before the origin answers, so + /// before the media type of the response is known. + #[must_use] + pub fn middleware_needs_whole_body(&self, settings: &Settings, path: &str) -> bool { + MiddlewarePhase::ALL.iter().any(|phase| { + settings + .phase_entries(*phase) + .entries() + .iter() + .filter(|entry| entry.covers_path(path)) + .flat_map(|entry| &entry.middleware) + .filter_map(|name| self.inner.middleware_named(name)) + .any(|middleware| middleware.handles_path(path) && middleware.needs_whole_body()) + }) + } + /// Every integration id the registry was built from, named or not, in /// registration order. A registry test enumerates this to check every /// builder was considered, while the deploy validation test iterates the diff --git a/crates/trusted-server-core/src/middleware.rs b/crates/trusted-server-core/src/middleware.rs index b67b033e99..dc0273bcc1 100644 --- a/crates/trusted-server-core/src/middleware.rs +++ b/crates/trusted-server-core/src/middleware.rs @@ -508,6 +508,15 @@ pub trait Middleware: Send + Sync { false } + /// Whether this middleware needs the origin's whole body. Where an entry + /// names it for a request's path, core asks the origin without the + /// reader's conditional and range headers, as it does for a page the ad + /// stack answers, because a `304` carries no body and a `206` only part of + /// one. `false` by default. + fn needs_whole_body(&self) -> bool { + false + } + /// The phases an entry may name this middleware in. fn phases(&self) -> &[MiddlewarePhase]; @@ -556,11 +565,16 @@ impl PhaseEntry { /// `path`. #[must_use] pub fn covers(&self, media_type: &str, path: &str) -> bool { - self.media_type == media_type - && self - .path - .as_deref() - .is_none_or(|prefix| path.starts_with(prefix)) + self.media_type == media_type && self.covers_path(path) + } + + /// Whether the entry covers a request for `path`, whatever the media type + /// of the response. + #[must_use] + pub fn covers_path(&self, path: &str) -> bool { + self.path + .as_deref() + .is_none_or(|prefix| path.starts_with(prefix)) } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4f0a2225e1..a5cbd3f77f 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -5281,10 +5281,13 @@ pub async fn handle_publisher_request( // a panic-prone invariant in the public request handler. let reader_compression = reader_compression.unwrap_or(Compression::None); - if should_run_ad_stack || personalization_requires_full_body { - // HTML document contexts whose output may be synthesized must not - // receive a cached 304 or partial 206. Non-document subresources contain - // no executable injected tag, so retain their validators and ranges. + let middleware_needs_whole_body = + integration_registry.middleware_needs_whole_body(settings, &request_path); + if should_run_ad_stack || personalization_requires_full_body || middleware_needs_whole_body { + // HTML document contexts whose output may be synthesized, and a + // middleware that needs the whole body, must not receive a cached 304 + // or partial 206. Other non-document subresources contain no executable + // injected tag, so retain their validators and ranges. strip_conditional_and_range_headers(&mut req); } @@ -21736,6 +21739,8 @@ mod tests { services: crate::platform::RuntimeServices, orchestrator: AuctionOrchestrator, response: PublisherResponse, + /// The origin, which keeps what it was asked. + origin: Arc, } /// A request through the publisher path, the origin answering `status`. @@ -21847,6 +21852,7 @@ mod tests { services, orchestrator, response, + origin: stub, } } @@ -22133,6 +22139,61 @@ mod tests { ); } + /// A reader's validators and range, as a crawler revalidating its copy + /// sends them. + const VALIDATORS_AND_RANGE: [(&str, &str); 4] = [ + ("if-none-match", "\"origin-etag\""), + ("if-modified-since", "Wed, 21 Oct 2015 07:28:00 GMT"), + ("range", "bytes=0-9"), + ("if-range", "\"origin-etag\""), + ]; + + /// The names of the headers the origin is asked with, for a `GET` of + /// `path` carrying [`VALIDATORS_AND_RANGE`], under settings refusing every + /// crawler. + async fn headers_the_origin_is_asked_with(path: &str) -> Vec { + let proxied = proxy_answered( + settings_refusing_every_crawler(), + publisher_request(Method::GET, path, &VALIDATORS_AND_RANGE), + 200, + b"User-agent: *\nAllow: /\n", + vec![("content-type", "text/plain")], + ) + .await; + proxied + .origin + .recorded_request_headers() + .concat() + .into_iter() + .map(|(name, _)| name) + .collect() + } + + /// The robots.txt middleware needs the origin's whole file, so the origin + /// is asked for it without the reader's validators and range, as for a + /// page the ad stack answers. A path no such middleware works on, the + /// longer path the entry covers among them, keeps them. + #[tokio::test] + async fn the_origin_is_asked_for_the_whole_robots_txt() { + let asked = headers_the_origin_is_asked_with(crate::robots_txt::ROBOTS_TXT_PATH).await; + for (name, _) in VALIDATORS_AND_RANGE { + assert!( + !asked.iter().any(|header| header == name), + "{name} should not reach the origin: {asked:?}" + ); + } + + for path in ["/humans.txt", "/robots.txt.bak"] { + let asked = headers_the_origin_is_asked_with(path).await; + for (name, _) in VALIDATORS_AND_RANGE { + assert!( + asked.iter().any(|header| header == name), + "{name} should reach the origin for {path}: {asked:?}" + ); + } + } + } + #[tokio::test] async fn a_file_the_origin_does_not_have_is_made_by_the_middleware_named_for_its_path() { use crate::integrations::registry_test_support::middleware_fixture as fixture; diff --git a/crates/trusted-server-core/src/robots_txt/middleware.rs b/crates/trusted-server-core/src/robots_txt/middleware.rs index 2efbd45eac..65d2cc92d3 100644 --- a/crates/trusted-server-core/src/robots_txt/middleware.rs +++ b/crates/trusted-server-core/src/robots_txt/middleware.rs @@ -87,6 +87,12 @@ impl Middleware for RobotsTxt { true } + // The origin is asked for its whole file, so a crawler's validators or + // range never bring a `304` or a `206` in place of the file. + fn needs_whole_body(&self) -> bool { + true + } + fn phases(&self) -> &[MiddlewarePhase] { &MiddlewarePhase::ALL } From 5266e31cb4b093b1f57a77326bbd27b9c970be6c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 11:56:56 +0100 Subject: [PATCH 10/23] Each adapter serves /robots.txt through the publisher path in a test No adapter test asked for `/robots.txt` once the route was removed, so the file's own `X-Robots-Tag`, a `HEAD` and a `POST` were not tested through an adapter. Each adapter's tests now ask a stub origin that answers each request in turn. A `GET` has the origin's file replaced by the refusal, a `HEAD` has the file made where the origin has none and gets no body, a `POST` keeps the origin's `404`, and each carries the tag. Axum, Cloudflare and Spin run their router with the stub as the origin and buffer the answer. Fastly runs its publisher fallback with its streaming finalizer, then writes the headers its finalizing middleware writes. --- .../tests/routes.rs | 132 ++++++++++++++++++ .../tests/routes.rs | 88 +++++++++++- .../trusted-server-adapter-fastly/src/app.rs | 112 +++++++++++++++ .../tests/routes.rs | 88 +++++++++++- 4 files changed, 412 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index 4e80a788b3..05857112c6 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -845,3 +845,135 @@ async fn health_carries_no_version() { "should write no version on /health, which the service_version middleware writes on a page" ); } + +// --------------------------------------------------------------------------- +// robots.txt +// --------------------------------------------------------------------------- + +/// The route test settings with a `[robots-txt]` section that refuses every +/// crawler and keeps one path open, and the entry that places the file. +fn settings_refusing_every_crawler() -> trusted_server_core::settings::Settings { + trusted_server_core::settings::Settings::from_toml( + r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "integration-test-proxy-secret" + + [ec] + module = "hmac" + + [ec.hmac] + passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true + + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "#, + ) + .expect("should parse robots.txt test settings") +} + +/// `/robots.txt` is the publisher path's, answered against an origin that +/// answers each request in turn. A `GET` has the origin's file replaced by +/// the refusal, a `HEAD` has the file made where the origin has none and +/// gets no body, and a `POST` keeps the origin's `404`, because a file is +/// made only for a `GET` or a `HEAD`. Each carries the refusal's +/// `X-Robots-Tag`, the file's own answer included. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn robots_txt_is_served_through_the_publisher_path_with_its_tag() { + use std::sync::Arc; + + use edgezero_core::http::Response; + use trusted_server_core::platform::PlatformHttpClient; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + for _ in 0..2 { + origin.push_response_with_headers( + 404, + b"not found".to_vec(), + vec![("content-type", "text/html")], + ); + } + let router = TrustedServerApp::routes_with_settings_and_services( + settings_refusing_every_crawler(), + build_services_with_http_client(Arc::clone(&origin) as Arc), + ) + .expect("should build router with a stub origin"); + let robots_txt = |method: &str| { + edgezero_core::http::request_builder() + .method(method) + .uri("https://test-publisher.example.com/robots.txt") + .header("host", "test-publisher.example.com") + .body(edgezero_core::body::Body::empty()) + .expect("should build request") + }; + let tag = |response: &Response| { + response + .headers() + .get("x-robots-tag") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + let body = |response: Response| { + let bytes = response + .into_body() + .into_bytes() + .expect("should hold the answer in memory"); + String::from_utf8(bytes.to_vec()).expect("the body is text") + }; + + let get = router + .oneshot(robots_txt("GET")) + .await + .expect("should serve the GET"); + assert_eq!(get.status(), 200); + assert_eq!(tag(&get).as_deref(), Some("noindex, nofollow")); + assert_eq!(body(get), "User-agent: *\nAllow: /ads.txt\nDisallow: /\n"); + + let head = router + .oneshot(robots_txt("HEAD")) + .await + .expect("should serve the HEAD"); + assert_eq!( + head.status(), + 200, + "the file is made where the origin has none" + ); + assert_eq!(tag(&head).as_deref(), Some("noindex, nofollow")); + assert_eq!(body(head), "", "a HEAD gets no body"); + + let post = router + .oneshot(robots_txt("POST")) + .await + .expect("should serve the POST"); + assert_eq!( + post.status(), + 404, + "a file is made only for a GET or a HEAD" + ); + assert_eq!(tag(&post).as_deref(), Some("noindex, nofollow")); + + assert_eq!( + origin.recorded_request_methods(), + ["GET", "HEAD", "POST"], + "each request reaches the origin" + ); +} diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index 824c8e9fa8..82f85138d0 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -576,8 +576,8 @@ fn every_fixed_route_beneath_an_underscore_prefix_is_reserved() { /// The route test settings with a `[robots-txt]` section that refuses every /// crawler and keeps one path open. -fn router_refusing_every_crawler() -> RouterService { - let settings = Settings::from_toml( +fn settings_refusing_every_crawler() -> Settings { + Settings::from_toml( r#" [publisher] domain = "test-publisher.example.com" @@ -604,9 +604,11 @@ fn router_refusing_every_crawler() -> RouterService { middleware = ["robots_txt"] "#, ) - .expect("should parse route test settings"); + .expect("should parse route test settings") +} - TrustedServerApp::routes_with_settings(settings) +fn router_refusing_every_crawler() -> RouterService { + TrustedServerApp::routes_with_settings(settings_refusing_every_crawler()) .expect("should build router from test settings") } @@ -653,6 +655,84 @@ async fn refusing_every_crawler_tags_every_response_and_registers_no_route() { ); } +/// `/robots.txt` is the publisher path's, answered against an origin that +/// answers each request in turn. A `GET` has the origin's file replaced by +/// the refusal, a `HEAD` has the file made where the origin has none and +/// gets no body, and a `POST` keeps the origin's `404`, because a file is +/// made only for a `GET` or a `HEAD`. Each carries the refusal's +/// `X-Robots-Tag`, the file's own answer included. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn robots_txt_is_served_through_the_publisher_path_with_its_tag() { + use std::sync::Arc; + + use trusted_server_core::platform::PlatformHttpClient; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + for _ in 0..2 { + origin.push_response_with_headers( + 404, + b"not found".to_vec(), + vec![("content-type", "text/html")], + ); + } + let router = TrustedServerApp::routes_with_settings_and_services( + settings_refusing_every_crawler(), + build_services_with_http_client(Arc::clone(&origin) as Arc), + ) + .expect("should build router with a stub origin"); + let robots_txt = |method: &str| { + request_builder() + .method(method) + .uri("https://test-publisher.example.com/robots.txt") + .header("host", "test-publisher.example.com") + .body(edgezero_core::body::Body::empty()) + .expect("should build request") + }; + let body = |response: Response| { + let bytes = response + .into_body() + .into_bytes() + .expect("should hold the answer in memory"); + String::from_utf8(bytes.to_vec()).expect("the body is text") + }; + + let get = route(router.clone(), robots_txt("GET")).await; + assert_eq!(get.status(), 200); + assert_eq!(robots_tag(&get).as_deref(), Some("noindex, nofollow")); + assert_eq!(body(get), "User-agent: *\nAllow: /ads.txt\nDisallow: /\n"); + + let head = route(router.clone(), robots_txt("HEAD")).await; + assert_eq!( + head.status(), + 200, + "the file is made where the origin has none" + ); + assert_eq!(robots_tag(&head).as_deref(), Some("noindex, nofollow")); + assert_eq!(body(head), "", "a HEAD gets no body"); + + let post = route(router, robots_txt("POST")).await; + assert_eq!( + post.status(), + 404, + "a file is made only for a GET or a HEAD" + ); + assert_eq!(robots_tag(&post).as_deref(), Some("noindex, nofollow")); + + assert_eq!( + origin.recorded_request_methods(), + ["GET", "HEAD", "POST"], + "each request reaches the origin" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_response_carries_no_blanket_rule_without_the_section() { let response = route( diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 705402aafd..af87235d51 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -3207,6 +3207,118 @@ mod tests { ); } + /// `/robots.txt` is the publisher path's, streamed against an origin that + /// answers each request in turn. A `GET` has the origin's file replaced + /// by the refusal, a `HEAD` has the file made where the origin has none + /// and gets no body, and a `POST` keeps the origin's `404`, because a + /// file is made only for a `GET` or a `HEAD`. Finalizing gives each the + /// refusal's `X-Robots-Tag`, the file's own answer included. + #[test] + fn robots_txt_is_served_through_the_publisher_path_with_its_tag() { + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + + let settings = Settings::from_toml( + r#" + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + module = "hmac" + + [ec.hmac] + passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true + + [request_signing] + enabled = false + + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "#, + ) + .expect("should parse robots.txt test settings"); + let state = build_state_from_settings(settings).expect("should build state"); + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + for _ in 0..2 { + origin.push_response_with_headers( + 404, + b"not found".to_vec(), + vec![("content-type", "text/html")], + ); + } + let services = + build_services_with_http_client(Arc::clone(&origin) as Arc); + let served = |method: Method| { + let request = request_builder() + .method(method) + .uri("https://test-publisher.com/robots.txt") + .header(header::HOST, "test-publisher.com") + .body(Body::empty()) + .expect("should build request"); + let mut response = block_on(super::dispatch_fallback(&state, &services, request)); + crate::middleware::apply_finalize_headers(&state.settings, None, &mut response); + response + }; + let tag = |response: &Response| { + response + .headers() + .get("x-robots-tag") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + let body = |response: Response| { + let bytes = block_on(response.into_body().into_bytes_bounded(1024 * 1024)) + .expect("should read the body"); + String::from_utf8(bytes.to_vec()).expect("the body is text") + }; + + let get = served(Method::GET); + assert_eq!(get.status(), StatusCode::OK); + assert_eq!(tag(&get).as_deref(), Some("noindex, nofollow")); + assert_eq!(body(get), "User-agent: *\nAllow: /ads.txt\nDisallow: /\n"); + + let head = served(Method::HEAD); + assert_eq!( + head.status(), + StatusCode::OK, + "the file is made where the origin has none" + ); + assert_eq!(tag(&head).as_deref(), Some("noindex, nofollow")); + assert_eq!(body(head), "", "a HEAD gets no body"); + + let post = served(Method::POST); + assert_eq!( + post.status(), + StatusCode::NOT_FOUND, + "a file is made only for a GET or a HEAD" + ); + assert_eq!(tag(&post).as_deref(), Some("noindex, nofollow")); + + assert_eq!( + origin.recorded_request_methods(), + ["GET", "HEAD", "POST"], + "each request reaches the origin" + ); + } + #[test] fn dispatch_runs_request_filter_and_threads_response_effects() { // Regression guard for the EdgeZero request-filter bypass: the publisher diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 7bae070ca4..9270184c02 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -914,8 +914,8 @@ fn every_fixed_route_beneath_an_underscore_prefix_is_reserved() { /// The route test settings with a `[robots-txt]` section that refuses every /// crawler and keeps one path open. -fn router_refusing_every_crawler() -> RouterService { - let settings = Settings::from_toml( +fn settings_refusing_every_crawler() -> Settings { + Settings::from_toml( r#" [publisher] domain = "test-publisher.example.com" @@ -942,9 +942,11 @@ fn router_refusing_every_crawler() -> RouterService { middleware = ["robots_txt"] "#, ) - .expect("should parse route test settings"); + .expect("should parse route test settings") +} - TrustedServerApp::routes_with_settings(settings) +fn router_refusing_every_crawler() -> RouterService { + TrustedServerApp::routes_with_settings(settings_refusing_every_crawler()) .expect("should build router from test settings") } @@ -991,6 +993,84 @@ async fn refusing_every_crawler_tags_every_response_and_registers_no_route() { ); } +/// `/robots.txt` is the publisher path's, answered against an origin that +/// answers each request in turn. A `GET` has the origin's file replaced by +/// the refusal, a `HEAD` has the file made where the origin has none and +/// gets no body, and a `POST` keeps the origin's `404`, because a file is +/// made only for a `GET` or a `HEAD`. Each carries the refusal's +/// `X-Robots-Tag`, the file's own answer included. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn robots_txt_is_served_through_the_publisher_path_with_its_tag() { + use std::sync::Arc; + + use trusted_server_core::platform::PlatformHttpClient; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + for _ in 0..2 { + origin.push_response_with_headers( + 404, + b"not found".to_vec(), + vec![("content-type", "text/html")], + ); + } + let router = TrustedServerApp::routes_with_settings_and_services( + settings_refusing_every_crawler(), + build_services_with_http_client(Arc::clone(&origin) as Arc), + ) + .expect("should build router with a stub origin"); + let robots_txt = |method: &str| { + request_builder() + .method(method) + .uri("https://test-publisher.example.com/robots.txt") + .header("host", "test-publisher.example.com") + .body(edgezero_core::body::Body::empty()) + .expect("should build request") + }; + let body = |response: Response| { + let bytes = response + .into_body() + .into_bytes() + .expect("should hold the answer in memory"); + String::from_utf8(bytes.to_vec()).expect("the body is text") + }; + + let get = route(router.clone(), robots_txt("GET")).await; + assert_eq!(get.status(), 200); + assert_eq!(robots_tag(&get).as_deref(), Some("noindex, nofollow")); + assert_eq!(body(get), "User-agent: *\nAllow: /ads.txt\nDisallow: /\n"); + + let head = route(router.clone(), robots_txt("HEAD")).await; + assert_eq!( + head.status(), + 200, + "the file is made where the origin has none" + ); + assert_eq!(robots_tag(&head).as_deref(), Some("noindex, nofollow")); + assert_eq!(body(head), "", "a HEAD gets no body"); + + let post = route(router, robots_txt("POST")).await; + assert_eq!( + post.status(), + 404, + "a file is made only for a GET or a HEAD" + ); + assert_eq!(robots_tag(&post).as_deref(), Some("noindex, nofollow")); + + assert_eq!( + origin.recorded_request_methods(), + ["GET", "HEAD", "POST"], + "each request reaches the origin" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_response_carries_no_blanket_rule_without_the_section() { let response = route( From a6327210c830575dd8f37db9e7c8daa419ae87e3 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 12:00:29 +0100 Subject: [PATCH 11/23] The changelog marks the removal of the robots.txt route as breaking The entry for the removed route did not carry the mark the other entries for removed configuration carry in the same block, yet a configuration with `[robots-txt]` and no `[[fetch]]` entry for the file is refused when it loads, and a build from before this change refuses the entry. The entry is marked and says that the two go live together. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca3f55e3c9..4fbc1c6300 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed -- The `/robots.txt` route on the four adapters, with `handle_robots_txt` and the route's own `Cache-Control: public, max-age=3600`. The file is made by the `robots_txt` middleware where an entry names it, and takes the deployment's cache policy for the path as every file does. +- **Breaking:** The `/robots.txt` route on the four adapters, with `handle_robots_txt` and the route's own `Cache-Control: public, max-age=3600`. The file is made by the `robots_txt` middleware where an entry names it, and takes the deployment's cache policy for the path as every file does. A configuration with `[robots-txt]` and no `[[fetch]]` entry naming `robots_txt` on `/robots.txt` is refused when it loads, and a build from before this change refuses that entry, so the new configuration and the new build go live together. - **Breaking:** HTTP Basic authentication is removed, with the `[[handlers]]` list that configured it. No request is asked for a password, and nothing is served beneath `/_ts/admin`. A configuration that still carries `[[handlers]]` is refused, with directions, when it is validated and when the service starts, so that no deployment believes a path is protected when it is not. A build before this change refuses a configuration without a handler covering `/_ts/admin`, and one with `[request_signing]` and no store ids, so no one configuration starts both builds: the new configuration and the new build go live together. A staged deploy reads a configuration key of its own (`ts deploy --staging`, `ts config push --staging`), which lets the pair be checked together first. Put a password on a path of the publisher's own at its origin or in the hosting platform. The `handler_password` secret is no longer read, and its entries in `fastly.toml`, the Spin manifest and the smoke scripts are removed. - **Breaking:** `POST /_ts/admin/cache/purge` is removed. It let anyone holding one shared password empty the service's cache over the publisher's own domain. Purge with `ts cache purge`, which asks the hosting platform. A request to the removed path is answered `404` by the server itself and is never forwarded to the origin. - The Fastly adapter's management API client is removed. The deployed service no longer reads a Fastly API token from a secret store named `api-keys`, which it used to write its own config store and secret store when a key was rotated. A deployment that linked that store can unlink it and revoke the token. `fastly.toml` and the Viceroy template drop their `api-keys` entries. From 5e255bb01250b1e4bd0601ca740d25729c31a774 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 12:00:55 +0100 Subject: [PATCH 12/23] The guides say where the robots.txt file takes the origin's place The configuration guide said that an answer other than a `404` or a page, such as a `401` from a staging site behind a login, is passed on as the origin gave it. It now says that the file is made in place of any such answer to a `GET` or `HEAD`, that the origin is asked for its whole file and that the middleware works on `/robots.txt` alone, as the code now does. The changelog entry for the middleware says the same, and a new entry and the integration guide describe `handles_path`, `writes_whole_document` and `needs_whole_body` for a module's own middleware. --- CHANGELOG.md | 3 ++- docs/guide/configuration.md | 6 +++--- docs/guide/integration-guide.md | 9 +++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fbc1c6300..2ea302eb39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,7 +85,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `robots_txt`, core's own middleware for the file `[robots-txt]` describes, named in a `[[fetch]]` entry for `text/plain` on `/robots.txt`. Its preparation makes the file from the selected modules through their holding, and its action replaces the origin's file with it, and the creating rule makes it where the origin answers `404`, or `200` with a page. A module that cannot answer with nothing held refuses the request with `503` and `Retry-After` before any document is processed. A section with no such entry, or with an entry naming the middleware on another path or media type, is refused when the settings load. +- `robots_txt`, core's own middleware for the file `[robots-txt]` describes, named in a `[[fetch]]` entry for `text/plain` on `/robots.txt`. Its preparation makes the file from the selected modules through their holding, and its action replaces the origin's file with it. The file is made in place of any other answer the origin gives to a `GET` or `HEAD`, such as a `404`, a page, an error or a redirect. The origin is asked for its whole file, without a crawler's conditional or range headers, and the middleware works on `/robots.txt` alone. A module that cannot answer with nothing held refuses the request with `503` and `Retry-After` before any document is processed. A section with no such entry, or with an entry naming the middleware on another path or media type, is refused when the settings load. +- A middleware may say which request paths it works on, with `Middleware::handles_path`, that it writes the whole document, with `writes_whole_document`, so core makes the document in place of any answer it would not hand the middleware as a `200` of the entry's media type, and that it needs the origin's whole body, with `needs_whole_body`, so the origin is asked without the reader's conditional and range headers. By default a middleware works on every path its entry covers and says neither of the other two. - A middleware may prepare, and may refuse. `Middleware::prepare` runs in the request's handler before the document is processed, as the middleware's module with the request's services, and what it answers is handed to `create` under the middleware's name, so a decision that needs a store or the network is made there while `create` stays synchronous. A refusal answers the request with its status and `Retry-After`, with no document, and nothing is stored. - A loading cache. `PlatformLoadingCache` in core loads a missing value once per key however many requests ask for it at the same moment, gives every waiting request the value or the error, and holds nothing that failed. A crate offers one through `IntegrationBuilder::with_loading_cache`, each feature that holds answers selects the module whose cache holds them in its own `[cache.]` table, and an adapter takes the selections with `loading_caches_from`. A feature that selects none takes the one loading cache module the build links, and gets `NoLoadingCache`, which loads every time, when the build links none or several. The new `crates/loading-cache/51degrees`, the module `loading-cache.51degrees`, implements it with the published `fiftyone-caching` package as a brief copy in process memory over the host's value cache or key-value store, and the stock list ships it, so every adapter takes it. Every load runs in its caller. - `[cache.robots-txt]` selects, with `module`, the loading cache that holds each robots.txt module's answer, and sets the lifetimes of its copies, in seconds: `memory_seconds` (default 60), `store_seconds` (default 604800) and `store_idle_seconds` (default 2592000). A memory lifetime may be 0, the others must be at least 1, and none may be more than a year. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index f4d9a4e4a6..6b2254abfc 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1054,7 +1054,7 @@ middleware = ["service_version"] ## robots.txt Configuration -What `/robots.txt` answers. With a `[robots-txt]` section, and a `[[fetch]]` entry for `text/plain` on the file's path naming `robots_txt`, core's own middleware makes the file, replacing the origin's own where it has one and making it where it has none. With no section the publisher keeps their own file and `/robots.txt` reaches the origin like any other path. +What `/robots.txt` answers. With a `[robots-txt]` section, and a `[[fetch]]` entry for `text/plain` on the file's path naming `robots_txt`, core's own middleware makes the file, replacing the origin's own where it has one and making it in place of any other answer. With no section the publisher keeps their own file and `/robots.txt` reaches the origin like any other path. ### `[robots-txt]` @@ -1076,7 +1076,7 @@ A module with settings of its own reads them from its own block, `[robots-txt. Date: Fri, 16 Oct 2026 12:07:52 +0100 Subject: [PATCH 13/23] The test of the made robots.txt file covers a 304 the origin sends An origin can answer `304` even when it is asked for its whole file. The test now sends one with the origin's `ETag`, which becomes the refusal with status `200` like the other answers the middleware is not handed whole. --- crates/trusted-server-core/src/publisher.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index a5cbd3f77f..29acfd4d12 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -22005,13 +22005,14 @@ mod tests { /// With `refuse_all` selected no other answer of the origin's takes the /// place of the file. A crawler takes an error, a redirect or an empty - /// answer as permission, and a file core cannot replace as the origin's - /// rules, so the refusal is made in place of each, with status `200`. + /// answer as permission, a `304` as leave to keep the copy it holds, and a + /// file core cannot replace as the origin's rules, so the refusal is made + /// in place of each, with status `200`. #[tokio::test] async fn robots_txt_is_made_in_place_of_any_answer_the_middleware_cannot_replace() { let permissive: &[u8] = b"User-agent: *\nAllow: /\n"; let plain = ("content-type", "text/plain"); - let answers: [(u16, &[u8], Vec<(&'static str, &'static str)>); 11] = [ + let answers: [(u16, &[u8], Vec<(&'static str, &'static str)>); 12] = [ (401, b"login", vec![("content-type", "text/html")]), (403, permissive, vec![plain]), (410, b"", vec![plain]), @@ -22039,6 +22040,7 @@ mod tests { b"User-agent: *\n", vec![plain, ("content-range", "bytes 0-13/23")], ), + (304, b"", vec![plain, ("etag", "\"origin-etag\"")]), ]; for (status, body, headers) in answers { From 5f3b4e8d59c7c990a80a3c36d692844886ea622e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 20:48:03 +0100 Subject: [PATCH 14/23] The Axum robots.txt test sends each request through a clone of its router Routing a request consumes the router, so the test's three requests, a GET, a HEAD and a POST, each go through a clone of it, as the adapter's other route tests do. --- crates/trusted-server-adapter-axum/tests/routes.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index 9cae76cb86..1dedb2c06b 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -1028,6 +1028,7 @@ async fn robots_txt_is_served_through_the_publisher_path_with_its_tag() { }; let get = router + .clone() .oneshot(robots_txt("GET")) .await .expect("should serve the GET"); @@ -1036,6 +1037,7 @@ async fn robots_txt_is_served_through_the_publisher_path_with_its_tag() { assert_eq!(body(get), "User-agent: *\nAllow: /ads.txt\nDisallow: /\n"); let head = router + .clone() .oneshot(robots_txt("HEAD")) .await .expect("should serve the HEAD"); @@ -1048,6 +1050,7 @@ async fn robots_txt_is_served_through_the_publisher_path_with_its_tag() { assert_eq!(body(head), "", "a HEAD gets no body"); let post = router + .clone() .oneshot(robots_txt("POST")) .await .expect("should serve the POST"); From 318530fc37ebe83074f071deafaec7182f4faa60 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 22:39:41 +0100 Subject: [PATCH 15/23] The robots.txt answers test names the type of one origin answer Clippy refuses the type of the test's twelve origin answers as too complex (`clippy::type_complexity` under `-D warnings`), so the test names the status, body and headers of one answer `OriginAnswer`, and the array is twelve of them. --- crates/trusted-server-core/src/publisher.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 450b10baa6..522d488cb4 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -22513,9 +22513,12 @@ mod tests { /// in place of each, with status `200`. #[tokio::test] async fn robots_txt_is_made_in_place_of_any_answer_the_middleware_cannot_replace() { + // The status, body and headers of one answer of the origin's. + type OriginAnswer = (u16, &'static [u8], Vec<(&'static str, &'static str)>); + let permissive: &[u8] = b"User-agent: *\nAllow: /\n"; let plain = ("content-type", "text/plain"); - let answers: [(u16, &[u8], Vec<(&'static str, &'static str)>); 12] = [ + let answers: [OriginAnswer; 12] = [ (401, b"login", vec![("content-type", "text/html")]), (403, permissive, vec![plain]), (410, b"", vec![plain]), From eb3c92b48b81273784c853671676936a594a64ca Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 23:41:18 +0100 Subject: [PATCH 16/23] Ask robots.txt contributors for the host the request was addressed to The robots.txt middleware asked its contributors for the publisher path's host, which follows Forwarded, then X-Forwarded-Host, then Host, and the holding keys each answer on the host the contributor was asked for. Axum and Cloudflare leave those headers in place, so a request carrying a forwarded host had a contributor asked for a host of its choosing, and was answered with one host's held rules on another host. The publisher path now reads the host the request was addressed to with http_util::addressed_host, from Host or the address and never from a forwarded header, and hands it to every preparation context. The middleware asks its contributors for that host, so the held answer is keyed on it, and a context that carries no such host asks no contributor and refuses the request. The refusal file asks nothing and is still made. The tests drive the publisher path with a contributor whose rule names the host it was asked for. Each of two hosts is answered for itself, and a request claiming another host is answered for the host it was addressed to, as is the plain request after it. --- .../trusted-server-core/src/module_context.rs | 31 +++ crates/trusted-server-core/src/publisher.rs | 7 + .../src/robots_txt/middleware.rs | 25 +- .../trusted-server-core/src/robots_txt/mod.rs | 217 +++++++++++++++++- 4 files changed, 270 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-core/src/module_context.rs b/crates/trusted-server-core/src/module_context.rs index e53ef8231e..cc896016e8 100644 --- a/crates/trusted-server-core/src/module_context.rs +++ b/crates/trusted-server-core/src/module_context.rs @@ -186,6 +186,15 @@ impl ResolvedRequest { self } + /// The same request for `host`, for an answer that is held under the host + /// it is made for, which must be the host the request was addressed to + /// and not one a forwarded header names. + #[must_use] + pub(crate) fn with_host(mut self, host: &str) -> Self { + host.clone_into(&mut self.host); + self + } + /// The borrowed view a context carries. #[must_use] pub fn view(&self) -> ModuleRequest<'_> { @@ -230,6 +239,9 @@ impl Copy for Gated<'_, T> {} #[derive(Clone, Copy)] pub struct ModuleContext<'r> { request: ModuleRequest<'r>, + /// The host the request was addressed to, which a forwarded header never + /// names, where core read it. + addressed_host: Option<&'r str>, evidence: Option<&'r dyn RequestInfo>, client: Option<&'r ClientInfo>, permissions: Option<&'r PermissionState>, @@ -250,6 +262,7 @@ impl<'r> ModuleContext<'r> { pub const fn new(request: ModuleRequest<'r>) -> Self { Self { request, + addressed_host: None, evidence: None, client: None, permissions: None, @@ -316,6 +329,18 @@ impl<'r> ModuleContext<'r> { } } + /// The same context carrying the host the request was addressed to, as + /// `http_util::addressed_host` reads it, for an answer held for every + /// request to that host. The request's own host may follow a forwarded + /// header, which a client can set to any host it likes. + #[must_use] + pub(crate) fn with_addressed_host(self, host: &'r str) -> Self { + Self { + addressed_host: Some(host), + ..self + } + } + /// The same context carrying the reader's location, whose use needs /// `requires`, being what the geo module that resolved it declares. #[must_use] @@ -484,6 +509,12 @@ impl<'c> ModuleCall<'c> { self.declared } + /// The host the request was addressed to, when the context carries it, + /// see [`ModuleContext::with_addressed_host`]. + pub(crate) fn addressed_host(&self) -> Option<&'c str> { + self.context.addressed_host + } + /// A gated value, when this module may be passed it. fn gated( &self, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 522d488cb4..cd40ef93b7 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -5133,6 +5133,10 @@ pub async fn handle_publisher_request( let request_info = RequestInfo::from_request(&req, services.client_info()); let request_host = &request_info.host; let request_scheme = &request_info.scheme; + // The host the request was addressed to, which a forwarded header never + // names. What a preparation holds for every request to a host is keyed + // on this, so one request cannot claim another host's answer. + let addressed_host = crate::http_util::addressed_host(&req); log::debug!( "Request info: host={}, scheme={} (X-Forwarded-Host: {:?}, Host: {:?}, X-Forwarded-Proto: {:?})", @@ -5821,6 +5825,7 @@ pub async fn handle_publisher_request( ); let module_context = ModuleContext::new(module_request) .with_settings(settings) + .with_addressed_host(&addressed_host) .with_request_state(ec_context, services); let stored_origin_host = settings.publisher.origin_host(); let document = TextResponse { @@ -6168,6 +6173,8 @@ pub async fn handle_publisher_request( ec_context, services, ); + let fetch_context = fetch_context.with_addressed_host(&addressed_host); + let serve_context = serve_context.with_addressed_host(&addressed_host); let document = TextResponse { content_type: &content_type, status, diff --git a/crates/trusted-server-core/src/robots_txt/middleware.rs b/crates/trusted-server-core/src/robots_txt/middleware.rs index 65d2cc92d3..941def7228 100644 --- a/crates/trusted-server-core/src/robots_txt/middleware.rs +++ b/crates/trusted-server-core/src/robots_txt/middleware.rs @@ -47,9 +47,11 @@ impl RobotsTxt { } /// The file for the request, which is the refusal alone or every selected - /// contributor's held answer assembled. + /// contributor's held answer assembled, each asked for `addressed_host`. + /// With no addressed host only the refusal can be made. async fn file( &self, + addressed_host: Option<&str>, request: ModuleRequest<'_>, settings: &Settings, services: &RuntimeServices, @@ -57,10 +59,21 @@ impl RobotsTxt { if self.config.refuses_all() { return Ok(document::refusal(&self.config.always_allow)); } - // The answers are held and served to every crawler, so the - // contributors are handed the request with nothing one crawler put in - // the address. - let request = ResolvedRequest::from_view(&request).without_query(); + // The answers are held for the host and served to every crawler of + // it, so the contributors are handed the request with nothing one + // crawler put in the address, for the host it was addressed to and + // never one a forwarded header names. The host is part of the key an + // answer is held under. + let Some(host) = addressed_host else { + return Err(Report::new(TrustedServerError::Configuration { + message: "robots.txt cannot be made without the host the request was \ + addressed to" + .to_owned(), + })); + }; + let request = ResolvedRequest::from_view(&request) + .without_query() + .with_host(host); file_for(&self.config, &self.selected, &request, services, settings).await } } @@ -103,7 +116,7 @@ impl Middleware for RobotsTxt { _context: &'a MiddlewareContext<'a>, ) -> Option> { Some(Box::pin(async move { - let file = match call.inject(self, Self::file) { + let file = match call.inject_with(self, call.addressed_host(), Self::file) { Ok(file) => file.await, Err(withheld) => Err(Report::new(TrustedServerError::Configuration { message: format!("robots.txt cannot be made for this request: {withheld:?}"), diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs index 7314c0d0df..6081424505 100644 --- a/crates/trusted-server-core/src/robots_txt/mod.rs +++ b/crates/trusted-server-core/src/robots_txt/mod.rs @@ -362,8 +362,8 @@ mod tests { use std::sync::Arc; use super::holding::tests::{ - ECHOED, Echoing, MemoryKvStore, Scripted, request, robots_request, rules, services_with, - settings, + ECHOED, Echoing, MemoryKvStore, PUBLISHER, Scripted, request, robots_request, rules, + services_with, settings, }; use super::*; use crate::integrations::{IntegrationBuilder, IntegrationRegistration, IntegrationRegistry}; @@ -584,10 +584,12 @@ mod tests { "the entry names the middleware" ); let request = robots_request(); + let addressed_host = crate::http_util::addressed_host(&request); let module_context = ModuleContext::new(ModuleRequest::of(&request, services.client_info())) .with_settings(settings) - .with_services(services); + .with_services(services) + .with_addressed_host(&addressed_host); let state = crate::integrations::IntegrationDocumentState::default(); let prepared = Prepared::default(); let context = crate::middleware::MiddlewareContext { @@ -624,10 +626,12 @@ mod tests { let settings = settings(); let services = services_with(Arc::default(), Arc::default()); let request = robots_request(); + let addressed_host = crate::http_util::addressed_host(&request); let module_context = ModuleContext::new(ModuleRequest::of(&request, services.client_info())) .with_settings(&settings) - .with_services(&services); + .with_services(&services) + .with_addressed_host(&addressed_host); let state = crate::integrations::IntegrationDocumentState::default(); let context = crate::middleware::test_support::context( crate::middleware::MiddlewarePhase::Fetch, @@ -783,6 +787,211 @@ mod tests { assert!(message.contains("allow_all"), "{message}"); } + /// A crawler's request for the file, addressed to `host`, carrying + /// `headers` as well. + fn robots_request_to(host: &str, headers: &[(&str, &str)]) -> http::Request { + let mut builder = http::Request::builder() + .uri(format!("https://{host}{ROBOTS_TXT_PATH}")) + .header(header::HOST, host); + for &(name, value) in headers { + builder = builder.header(name, value); + } + builder + .body(EdgeBody::empty()) + .expect("should build the request") + } + + /// The file the publisher path answers `request` with, under `settings` + /// with the echoing contributor registered, while `origin` answers `404` + /// so the middleware makes the file. + async fn file_through_the_publisher_path( + settings: &Settings, + services: &RuntimeServices, + origin: &crate::platform::test_support::StubHttpClient, + request: http::Request, + ) -> String { + let registry = IntegrationRegistry::with_registrations(settings, &[echo_builder()]) + .expect("should build the registry"); + let orchestrator = crate::auction::AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(settings), + ); + origin.push_response_with_headers( + 404, + b"not found".to_vec(), + vec![("content-type", "text/html")], + ); + let method = request.method().clone(); + let answer = crate::publisher::handle_publisher_request( + crate::publisher::AppContext { + settings, + integration_registry: ®istry, + }, + services, + None, + &mut crate::ec::EcContext::default(), + crate::publisher::AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + crate::cache_policy::EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should answer the request"); + let response = crate::publisher::buffer_publisher_response_async( + answer, + &method, + settings, + ®istry, + &orchestrator, + services, + ) + .await + .expect("should make the file"); + assert_eq!(response.status(), StatusCode::OK); + let body = response + .into_body() + .into_bytes_bounded(4096) + .await + .expect("should read the file"); + String::from_utf8(body.to_vec()).expect("the file is text") + } + + /// Services asking `origin`, holding answers in memory, as one instance + /// of a host does. + fn services_asking( + origin: &Arc, + ) -> RuntimeServices { + crate::platform::test_support::build_services_with_http_client( + Arc::clone(origin) as Arc + ) + .with_kv_store(Arc::new(MemoryKvStore::default())) + .with_loading_cache(Arc::new( + crate::platform::test_support::MemoryLoadingCache::default(), + )) + } + + /// One deployment may serve several hosts, and each is answered with + /// what its contributors gave for that host, never with an answer held + /// for another. + #[tokio::test] + async fn each_host_is_answered_for_itself() { + let settings = settings_selecting( + serde_json::json!({ "modules": ["echo"] }), + ECHO_BUILDER_MODULE, + ); + let origin = Arc::new(crate::platform::test_support::StubHttpClient::new()); + let services = services_asking(&origin); + + for host in [PUBLISHER, "other.publisher.example"] { + let file = file_through_the_publisher_path( + &settings, + &services, + &origin, + robots_request_to(host, &[]), + ) + .await; + + assert!( + file.contains(&format!("Disallow: /{host}/robots.txt?")), + "{host} should be answered for itself: {file}" + ); + } + } + + /// `Forwarded` and `X-Forwarded-Host` name whatever host a client likes, + /// and the publisher path's own host follows them, so a contributor is + /// asked for the host the request was addressed to, and an answer made + /// for a claimed host is never held for the real one. + #[tokio::test] + async fn a_forwarded_host_is_never_the_host_a_contributor_is_asked_for() { + let settings = settings_selecting( + serde_json::json!({ "modules": ["echo"] }), + ECHO_BUILDER_MODULE, + ); + let origin = Arc::new(crate::platform::test_support::StubHttpClient::new()); + let services = services_asking(&origin); + let claiming = robots_request_to( + PUBLISHER, + &[ + ("forwarded", "host=elsewhere.example"), + ("x-forwarded-host", "elsewhere.example"), + ], + ); + + let claimed = + file_through_the_publisher_path(&settings, &services, &origin, claiming).await; + let after = file_through_the_publisher_path( + &settings, + &services, + &origin, + robots_request_to(PUBLISHER, &[]), + ) + .await; + + for file in [claimed, after] { + assert!( + !file.contains("elsewhere.example"), + "a claimed host should never reach the file: {file}" + ); + assert!( + file.contains(&format!("Disallow: /{PUBLISHER}/robots.txt?")), + "the file should be made for the host addressed: {file}" + ); + } + } + + /// An answer is held for the host the request was addressed to, so a + /// call that does not carry that host asks no contributor and refuses + /// the request. The refusal asks nothing, so it is still made. + #[tokio::test] + async fn without_the_addressed_host_only_the_refusal_is_made() { + let scripted = Arc::new(Scripted::new(vec![Ok(rules("User-Agent: *\nAllow: /"))])); + let asking = middleware::RobotsTxt::new( + parsed(serde_json::json!({ "modules": ["scripted"] })), + vec![( + "scripted", + Some(Arc::clone(&scripted) as Arc), + )], + ); + let refusing = middleware::RobotsTxt::new(RobotsTxtConfig::refuse_all(), Vec::new()); + let settings = settings(); + let services = services_with(Arc::default(), Arc::default()); + let request = robots_request(); + let module_context = + ModuleContext::new(ModuleRequest::of(&request, services.client_info())) + .with_settings(&settings) + .with_services(&services); + let state = crate::integrations::IntegrationDocumentState::default(); + let context = crate::middleware::test_support::context( + crate::middleware::MiddlewarePhase::Fetch, + &state, + ); + let call = module_context.call( + middleware::MODULE, + crate::permissions::PermissionSet::none(), + ); + + let refusal = asking + .prepare(call, &context) + .expect("the middleware prepares") + .await + .expect_err("no contributor is asked without the host"); + let file = refusing + .prepare(call, &context) + .expect("the middleware prepares") + .await + .expect("the refusal is made without the host"); + + assert_eq!(refusal.status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(scripted.times_asked(), 0, "no contributor should be asked"); + assert_eq!( + *file.downcast::().expect("the file is text"), + "User-agent: *\nDisallow: /\n" + ); + } + /// A contributor with nothing to add, which answers a comment and no /// group. #[derive(Debug)] From aefbfb17a05b93e1ccdbe071cf42a9b7c7d822a8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 23:43:40 +0100 Subject: [PATCH 17/23] Test on Axum that a forwarded host never reaches a robots.txt contributor Axum leaves Forwarded and X-Forwarded-Host in place, so this adapter is where a claimed host would reach a contributor. The test sends a request for /robots.txt addressed to the publisher's host with both headers naming another host, through the router with the probe's contributor selected and an origin that has no file. The probe's rule names the host it was asked for, and the file names the host the request was addressed to and never the claimed one. TrustedServerApp::routes_with_registrations_and_services builds the router with both the deployment's own builders and the services every request is handed, which the test needs to register the probe and stub the origin. --- crates/trusted-server-adapter-axum/src/app.rs | 19 ++++++ .../tests/seam_probe.rs | 65 +++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 2852845962..c47e35748a 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -774,6 +774,25 @@ impl TrustedServerApp { let state = build_state_with_services(settings, Some(services))?; Ok(build_router(&state)) } + + /// Build the full router as + /// [`TrustedServerApp::routes_with_registrations`] does, with every + /// request handed a clone of `services`, as + /// [`TrustedServerApp::routes_with_settings_and_services`] hands them. + /// + /// # Errors + /// + /// Returns the errors [`TrustedServerApp::routes_with_registrations`] + /// returns. + pub fn routes_with_registrations_and_services( + settings: Settings, + integrations: &[IntegrationBuilder], + services: RuntimeServices, + ) -> Result> { + let state = + build_state_with_registrations_and_services(settings, integrations, Some(services))?; + Ok(build_router(&state)) + } } /// Answers the attestation pages without the request preparers, because diff --git a/crates/trusted-server-adapter-axum/tests/seam_probe.rs b/crates/trusted-server-adapter-axum/tests/seam_probe.rs index 5396e6c5a5..380e7cd17a 100644 --- a/crates/trusted-server-adapter-axum/tests/seam_probe.rs +++ b/crates/trusted-server-adapter-axum/tests/seam_probe.rs @@ -334,6 +334,71 @@ fn robots_txt_selection_naming_a_module_that_is_not_running_fails_at_startup() { ); } +/// This adapter leaves `Forwarded` and `X-Forwarded-Host` in place, and the +/// probe's robots.txt rule names the host it was asked for, so the file +/// shows that a claimed host never reaches a contributor. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_forwarded_host_never_reaches_a_robots_txt_module() { + use trusted_server_core::platform::PlatformHttpClient; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + + let settings = settings_with(&format!( + r#" + [robots-txt] + modules = ["{module}"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + {PROBE_BLOCK} + "#, + module = seam_probe::module_name(), + )); + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 404, + b"not found".to_vec(), + vec![("content-type", "text/html")], + ); + let router = TrustedServerApp::routes_with_registrations_and_services( + settings, + &[seam_probe::builder()], + build_services_with_http_client(Arc::clone(&origin) as Arc), + ) + .expect("should build the router with a stub origin"); + let mut service = EdgeZeroAxumService::new(router); + let request = Request::builder() + .method("GET") + .uri("/robots.txt") + .header("host", "test-publisher.example.com") + .header("x-forwarded-host", "elsewhere.example") + .header("forwarded", "host=elsewhere.example") + .body(AxumBody::empty()) + .expect("should build request"); + + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should respond"); + assert_eq!(response.status().as_u16(), 200); + let file = body_text(response).await; + + assert!( + file.contains("Disallow: /test-publisher.example.com/robots.txt?"), + "the probe should be asked for the host addressed: {file}" + ); + assert!( + !file.contains("elsewhere.example"), + "a claimed host should never reach the file: {file}" + ); +} + /// A request prepares exactly once whether it is served by a named route or by /// the fallback, so the invariant holds across the whole route table rather /// than only on the path the probe's own proxy route sits on. From 0869a85c8c19450e51ba89f8dfb3fc8afca7e37b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 23:46:43 +0100 Subject: [PATCH 18/23] Answer robots.txt on Fastly before the Edge Cookie steps and the filters From the robots.txt middleware on, a request for the file took the publisher fallback as any request does, through the request preparers, then the Edge Cookie request state, which reads the crawler's cookies and consent signals and asks the geo module for its location, then the request filters, and the Edge Cookie finalize step on the way out. A filter that answers, such as a bot challenge, stood in for the file, and a crawler reads a 4xx for robots.txt as permission to crawl everything. With [robots-txt] in the settings, a GET or HEAD of /robots.txt now goes to the publisher path once the modules have prepared it, with an empty Edge Cookie context and no auction, and no Edge Cookie state rides on the response. The origin is still asked and the middleware still makes the file. Any other method on the path takes the fallback as before. The test registers a module whose filter challenges every request. Another path and a POST of /robots.txt get the challenge, and a GET of /robots.txt gets the file the middleware made from the origin's, with no Edge Cookie state, the origin asked once. --- .../trusted-server-adapter-fastly/src/app.rs | 60 ++++++++ .../src/app/seam_probe_tests.rs | 139 ++++++++++++++++++ 2 files changed, 199 insertions(+) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 0fc9cb42d9..026497cf06 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1017,6 +1017,14 @@ async fn dispatch_fallback( return http_error(&report); } + // A crawler asking for robots.txt is answered once the modules have + // prepared the request, before the Edge Cookie steps and the request + // filters, so nothing is read from its cookies or written back, and no + // filter's answer stands in for the file. + if asks_for_robots_txt(&state.settings, &method, &path) { + return answer_robots_txt(state, services, req, &method).await; + } + let mut ec = build_ec_request_state(&state.settings, services, &req).await; if let Some(report) = ec.setup_error.take() { let response = http_error(&report); @@ -1149,6 +1157,58 @@ async fn dispatch_fallback( attach_dispatch_extensions(response, ec, effects) } +/// Whether the request is a crawler's `GET` or `HEAD` of the robots.txt that +/// `[robots-txt]` makes. +fn asks_for_robots_txt(settings: &Settings, method: &Method, path: &str) -> bool { + settings.robots_txt.is_some() + && path == trusted_server_core::robots_txt::ROBOTS_TXT_PATH + && matches!(*method, Method::GET | Method::HEAD) +} + +/// Answers robots.txt through the publisher path with an empty Edge Cookie +/// context and no auction, so none of the crawler's cookies or consent +/// signals are read, and attaches no Edge Cookie state, so the finalize step +/// writes nothing back. +async fn answer_robots_txt( + state: &AppState, + services: &RuntimeServices, + req: Request, + method: &Method, +) -> Response { + let answer = handle_publisher_request( + AppContext { + settings: &state.settings, + integration_registry: state.registry.as_ref(), + }, + services, + None, + &mut EcContext::default(), + AuctionDispatch { + orchestrator: &state.orchestrator, + slots: &[], + registry: None, + }, + req, + EdgeCacheHeader::SurrogateControl, + ) + .await; + let response = match answer { + Ok(answer) => { + publisher_response_into_streaming_response( + answer, + method, + Arc::clone(&state.settings), + state.registry.as_ref(), + Arc::clone(&state.orchestrator), + services.clone(), + ) + .await + } + Err(report) => Err(report), + }; + response.unwrap_or_else(|report| http_error(&report)) +} + /// Returns `true` when an asset response should carry a (streamed) body. /// /// `HEAD` responses and bodiless statuses (204, 304) advertise the origin diff --git a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs index d900fe7524..83cea0e348 100644 --- a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs @@ -270,6 +270,145 @@ fn no_blanket_rule_is_placed_on_responses_without_the_section() { ); } +/// A module whose request filter answers every request with a challenge, as +/// bot protection does. +mod challenge { + use std::sync::Arc; + + use edgezero_core::body::Body; + use edgezero_core::http::{Response, StatusCode}; + use error_stack::Report; + use trusted_server_core::error::TrustedServerError; + use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationRegistration, IntegrationRequestFilter, + RequestFilterDecision, RequestFilterEffects, RequestFilterInput, + }; + use trusted_server_core::settings::Settings; + + /// The module's id. + const ID: &str = "robots-txt-challenge"; + + /// Answers every request with a `403` challenge. + struct Challenging; + + #[async_trait::async_trait(?Send)] + impl IntegrationRequestFilter for Challenging { + fn integration_id(&self) -> &'static str { + ID + } + + async fn filter_request( + &self, + _input: RequestFilterInput<'_>, + ) -> Result> { + let mut response = Response::new(Body::from("a challenge")); + *response.status_mut() = StatusCode::FORBIDDEN; + Ok(RequestFilterDecision::Respond { + response: Box::new(response), + effects: RequestFilterEffects::default(), + }) + } + } + + fn register( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(ID) + .with_request_filter(Arc::new(Challenging)) + .build(), + )) + } + + fn accept(_settings: &Settings) -> Result> { + Ok(true) + } + + /// The module, selected by `[testing] modules = ["robots-txt-challenge"]`. + pub(super) fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(ID, ID, register, accept) + .with_module_name("testing.robots-txt-challenge") + } +} + +/// A crawler asking for robots.txt is answered before the Edge Cookie steps +/// and the request filters, so a filter that answers every other request +/// with a challenge never stands in for the file, and no Edge Cookie state +/// rides on the answer. Another method on the path is no crawler's request +/// for the file, so the filter answers it. +#[test] +fn robots_txt_is_answered_before_the_edge_cookie_steps_and_the_filters() { + use trusted_server_core::platform::PlatformHttpClient; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + + let state = state_with( + settings_with(&format!( + r#" + [robots-txt] + modules = "refuse_all" + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + {HMAC_BLOCK} + + [testing] + modules = ["robots-txt-challenge"] + "# + )), + &[challenge::builder()], + ); + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + let services = + build_services_with_http_client(Arc::clone(&origin) as Arc); + let dispatch = |method: Method, path: &str| { + let request = request_builder() + .method(method) + .uri(format!("https://test-publisher.com{path}")) + .body(Body::empty()) + .expect("should build request"); + block_on(super::dispatch_fallback(&state, &services, request)) + }; + + let page = dispatch(Method::GET, "/page"); + assert_eq!( + page.status().as_u16(), + 403, + "the filter should answer another request" + ); + let post = dispatch(Method::POST, "/robots.txt"); + assert_eq!( + post.status().as_u16(), + 403, + "the filter should answer a method other than GET and HEAD" + ); + + let file = dispatch(Method::GET, "/robots.txt"); + assert_eq!( + file.status().as_u16(), + 200, + "the file should be served, not the filter's challenge" + ); + assert!( + file.extensions().get::().is_none(), + "no Edge Cookie state should ride on the file" + ); + assert_eq!(body_text(file), "User-agent: *\nDisallow: /\n"); + assert_eq!( + origin.recorded_request_methods(), + ["GET"], + "only the crawler's request for the file should reach the origin" + ); +} + /// A request prepares exactly once whether a named route or the fallback /// serves it. #[test] From 55321838d1ceaa2dda9109977fab7b5c6bb46ea8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Oct 2026 23:48:20 +0100 Subject: [PATCH 19/23] Restore what the docs say of robots.txt's host and Fastly's order The robots.txt middleware dropped two statements because they had stopped being true. A contributor is again asked for the host the request was addressed to, never one Forwarded or X-Forwarded-Host names, and on Fastly the file is again answered before the Edge Cookie steps and the request filters. The contributor's documentation and the configuration guide say so once more. The changelog entry for [robots-txt] kept both statements, so it is true again with no change. --- crates/trusted-server-core/src/robots_txt/contributor.rs | 7 ++++--- docs/guide/configuration.md | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/robots_txt/contributor.rs b/crates/trusted-server-core/src/robots_txt/contributor.rs index 058ebd4374..773ec0560f 100644 --- a/crates/trusted-server-core/src/robots_txt/contributor.rs +++ b/crates/trusted-server-core/src/robots_txt/contributor.rs @@ -63,9 +63,10 @@ pub trait RobotsTxtContributor: Send + Sync + core::fmt::Debug { /// Core holds the answer for the publisher and the host the request was /// addressed to, and serves it to every crawler of that host. So the call /// carries the request the answer was asked for, being its method, path, - /// host and scheme without its query, and the request's services with no - /// client information and no host signals, and nothing about who asked. - /// An implementation hands its own function to [`ModuleCall::inject`], + /// host and scheme without its query, with the host never taken from a + /// forwarded header, and the request's services with no client + /// information and no host signals, and nothing about who asked. An + /// implementation hands its own function to [`ModuleCall::inject`], /// naming what it needs. Its answer must be the same for every request /// for that host, because it is served for all of them. /// diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 4ba4552e8d..cd5ff4010a 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1073,10 +1073,11 @@ A module with settings of its own reads them from its own block, `[robots-txt. Date: Sat, 17 Oct 2026 00:11:33 +0100 Subject: [PATCH 20/23] The robots.txt failure test plants a word no function name holds The test checks that a contributor's failure is logged without the parts of an address that can carry a key, by planting the word "secret" in each one. The report it reads carries a backtrace, and on Windows the backtrace named a neighbouring function in the secret_resolution tests, so the test failed there. The planted word is now one no function name holds. --- crates/trusted-server-core/src/robots_txt/holding.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/robots_txt/holding.rs b/crates/trusted-server-core/src/robots_txt/holding.rs index 44a14217cd..ed8e2bccf2 100644 --- a/crates/trusted-server-core/src/robots_txt/holding.rs +++ b/crates/trusted-server-core/src/robots_txt/holding.rs @@ -934,9 +934,9 @@ pub(crate) mod tests { #[tokio::test] async fn a_failure_gives_only_the_host_of_an_address_it_names() { let scripted = Arc::new(Scripted::new(vec![Err( - "GET https://user:secret-password@rules.example:8443/v1/secret-in-path/rules\ - ?licence=secret-in-query#secret-in-fragment failed, and \"http://other.example/\ - secret-in-quotes\" too" + "GET https://user:xyzzy-password@rules.example:8443/v1/xyzzy-in-path/rules\ + ?licence=xyzzy-in-query#xyzzy-in-fragment failed, and \"http://other.example/\ + xyzzy-in-quotes\" too" .to_owned(), )])); let services = services_with(Arc::default(), Arc::default()); @@ -957,7 +957,9 @@ pub(crate) mod tests { "{logged}" ); assert!(logged.contains("http://other.example"), "{logged}"); - assert!(!logged.contains("secret"), "{logged}"); + // A word no function name holds, because the report carries a + // backtrace when one is captured. + assert!(!logged.contains("xyzzy"), "{logged}"); } /// A held copy that no longer reads is removed and the contributor asked From 76e0ec626d8ed3ab8f29e5ff3e6b0c2c1c727ab7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 17 Oct 2026 00:46:09 +0100 Subject: [PATCH 21/23] The Fastly robots.txt order test sends the Host header a request carries The test's requests named their host in the address alone. Core reads a page's host from the request's headers and runs no middleware for a request that names none, so the made file never replaced the origin's and the test failed. Every request a crawler sends carries Host, and the test now does too. All 272 tests of the Fastly adapter pass under Viceroy. --- crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs index 83cea0e348..4fa833033b 100644 --- a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs @@ -373,6 +373,7 @@ fn robots_txt_is_answered_before_the_edge_cookie_steps_and_the_filters() { let request = request_builder() .method(method) .uri(format!("https://test-publisher.com{path}")) + .header("host", "test-publisher.com") .body(Body::empty()) .expect("should build request"); block_on(super::dispatch_fallback(&state, &services, request)) From bcbc92b2611f3a90b801891ec77756acfd0ffa02 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 28 Oct 2026 11:29:02 +0000 Subject: [PATCH 22/23] The Fastly robots.txt test's settings carry the sample policy --- crates/trusted-server-adapter-fastly/src/app.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 4d6630f333..a842cd7a69 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -3157,7 +3157,9 @@ mod tests { middleware = ["robots_txt"] "#, ) - .expect("should parse robots.txt test settings"); + .expect("should parse robots.txt test settings") + // A loaded domain carries its policy. + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()); let state = build_state_from_settings(settings).expect("should build state"); let origin = Arc::new(StubHttpClient::new()); origin.push_response_with_headers( From 6b0a0f6f452a7b2aa9b4588b0baea5939825a36d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 28 Oct 2026 11:29:03 +0000 Subject: [PATCH 23/23] The Axum robots.txt test's settings carry the sample policy --- crates/trusted-server-adapter-axum/tests/routes.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index 001c7aaedd..2052385107 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -954,6 +954,7 @@ fn settings_refusing_every_crawler() -> trusted_server_core::settings::Settings "#, ) .expect("should parse robots.txt test settings") + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()) } /// `/robots.txt` is the publisher path's, answered against an origin that