From 31dd42d80765b6c79ced1a1e6cb28901a8314357 Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 9 Oct 2026 16:13:38 +0100
Subject: [PATCH 01/15] A module can add rules to robots.txt, and core writes
 the file

`[robots-txt] modules` names, in order, what makes the file. `refuse_all`
and `allow_all` are core's own and need nothing else. Any other name is a
contributor a module declared with `with_robots_txt_contributor`, and core
names none of them.

A contributor returns groups of records and core writes every byte. That is
what lets these hold whatever a contributor returns.

- With `refuse_all` selected the file is the refusal and nothing else.
- A path in `always_allow` stays open to every crawler a contribution
  refuses the whole site to, by a group of core's own below the
  contributions, which are not edited.
- The `Sitemap` line and the publisher's own text go around the
  contributions.

Each contributor's answer is held in the key-value store, under a key that
is a hash of the publisher, the contributor's name and the contributor's
fingerprint, and the contributor is asked again when its own refresh is due.
A contributor that fails has its held answer served past its age. With
nothing held the file fails closed with `503`, because leaving a
contributor's rules out would allow by omission what those rules refused,
and a `404` or an empty file reads to a crawler as permission. A store that
cannot give an entry a lifetime is written without one, so the answer is
still there to serve.

A contributor is asked through the module context, with the request's
method, path, host and scheme and without its query or anything about who
asked, because its answer is served to every crawler.

`[robots-txt]` is refused when the settings load if `modules` is absent,
empty or names one thing twice, if a key is not a setting, or if a block
belongs to a contributor `modules` does not name. A name nothing supplies is
refused when the registry is built, so the file does not fail on the first
request for it.
---
 .../src/integrations/registry.rs              | 100 +++
 crates/trusted-server-core/src/lib.rs         |   1 +
 .../trusted-server-core/src/module_context.rs |   9 +
 .../src/robots_txt/contributor.rs             |  85 ++
 .../src/robots_txt/document.rs                | 461 ++++++++++
 .../src/robots_txt/holding.rs                 | 650 ++++++++++++++
 .../trusted-server-core/src/robots_txt/mod.rs | 827 ++++++++++++++++++
 crates/trusted-server-core/src/settings.rs    | 100 +++
 8 files changed, 2233 insertions(+)
 create mode 100644 crates/trusted-server-core/src/robots_txt/contributor.rs
 create mode 100644 crates/trusted-server-core/src/robots_txt/document.rs
 create mode 100644 crates/trusted-server-core/src/robots_txt/holding.rs
 create mode 100644 crates/trusted-server-core/src/robots_txt/mod.rs

diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs
index c7758b0d39..fb37af1a1e 100644
--- a/crates/trusted-server-core/src/integrations/registry.rs
+++ b/crates/trusted-server-core/src/integrations/registry.rs
@@ -21,6 +21,7 @@ use crate::http_util::is_navigation_request;
 use crate::middleware::{Middleware, MiddlewareChain, MiddlewarePhase, PhaseEntries};
 use crate::module_context::{ModuleCall, ModuleContext, ResolvedRequest};
 use crate::platform::{DisabledGeo, PlatformGeo, RuntimeServices};
+use crate::robots_txt::{ALLOW_ALL, REFUSE_ALL, RobotsTxtContributor};
 use crate::settings::Settings;
 
 /// What a middleware's element handler decides about one attribute of an
@@ -672,6 +673,12 @@ pub struct IntegrationRegistration {
     /// Declaring one does not make it active, because the module is only asked
     /// to classify a request when `[device] module` names it.
     pub device_module: Option<(&'static str, Arc<dyn DeviceModule>)>,
+    /// The rules this registration adds to `/robots.txt`, with the name
+    /// `[robots-txt] modules` selects them by.
+    ///
+    /// Declaring them does not put them in the file, because the file is only
+    /// built from the contributors `[robots-txt] modules` names.
+    pub robots_txt_contributor: Option<(&'static str, Arc<dyn RobotsTxtContributor>)>,
 }
 
 impl IntegrationRegistration {
@@ -701,6 +708,7 @@ impl IntegrationRegistrationBuilder {
                 geo_module: None,
                 ec_module: None,
                 device_module: None,
+                robots_txt_contributor: None,
             },
         }
     }
@@ -782,6 +790,21 @@ impl IntegrationRegistrationBuilder {
         self
     }
 
+    /// Declare the rules this registration adds to `/robots.txt`, under their
+    /// `name`.
+    ///
+    /// They are only asked for when `[robots-txt] modules` names them. Core
+    /// holds what they return and writes the file, see [`crate::robots_txt`].
+    #[must_use]
+    pub fn with_robots_txt_contributor(
+        mut self,
+        name: &'static str,
+        contributor: Arc<dyn RobotsTxtContributor>,
+    ) -> Self {
+        self.registration.robots_txt_contributor = Some((name, contributor));
+        self
+    }
+
     /// Mark this integration's JS module for deferred loading via
     /// `<script defer>` instead of the main synchronous bundle.
     #[must_use]
@@ -882,6 +905,10 @@ struct IntegrationRegistryInner {
     // Device modules the running registrations supply, each with its name, in
     // registration order. `[device] module` picks at most one of them.
     device_modules: Vec<(&'static str, Arc<dyn DeviceModule>)>,
+    // The robots.txt contributors the running registrations supply, each with
+    // its name, in registration order. `[robots-txt] modules` picks any of
+    // them, in its own order.
+    robots_txt_contributors: Vec<(&'static str, Arc<dyn RobotsTxtContributor>)>,
     // Every builder's module name, with whether a section selects it, so the
     // refusal of a name a selector wrote can say what that name is.
     builder_modules: Vec<(&'static str, bool)>,
@@ -919,6 +946,7 @@ impl Default for IntegrationRegistryInner {
             geo_modules: Vec::new(),
             ec_modules: Vec::new(),
             device_modules: Vec::new(),
+            robots_txt_contributors: Vec::new(),
             builder_modules: Vec::new(),
             geo_module: None,
             ec_module: None,
@@ -949,6 +977,7 @@ const DEVICE_MODULE_FASTLY: &str = "fastly";
 const EDGECOOKIE_TYPE: &str = crate::ec::module::MODULE_TYPE;
 const DEVICE_TYPE: &str = "device";
 const GEO_TYPE: &str = "geo";
+const ROBOTS_TXT_TYPE: &str = crate::robots_txt::MODULE_TYPE;
 
 impl IntegrationRegistryInner {
     /// The registered middleware an entry means by `name`.
@@ -1191,6 +1220,48 @@ fn resolve_device_module(
     Ok(resolved)
 }
 
+/// Refuses a `[robots-txt] modules` that names something nothing this
+/// deployment runs supplies.
+///
+/// Only knowable here, where every registration has been handed over, so it
+/// is checked when the registry is built rather than when the settings load.
+/// The file would otherwise fail on the first request for it rather than at
+/// deploy.
+fn check_robots_txt_selection(
+    settings: &Settings,
+    inner: &IntegrationRegistryInner,
+) -> Result<(), Report<TrustedServerError>> {
+    let Some(robots_txt) = settings.robots_txt.as_ref() else {
+        return Ok(());
+    };
+    for written in &robots_txt.modules {
+        if written == REFUSE_ALL
+            || written == ALLOW_ALL
+            || find_named(ROBOTS_TXT_TYPE, written, &inner.robots_txt_contributors).is_some()
+        {
+            continue;
+        }
+        let offered: Vec<&str> = [REFUSE_ALL, ALLOW_ALL]
+            .into_iter()
+            .chain(
+                inner
+                    .robots_txt_contributors
+                    .iter()
+                    .map(|(name, _)| crate::module_name::short_form(ROBOTS_TXT_TYPE, name)),
+            )
+            .collect();
+        return Err(Report::new(TrustedServerError::Configuration {
+            message: format!(
+                "`[robots-txt] modules` names `{written}`, which nothing this deployment runs \
+                 supplies. What it can name is [{}]{}",
+                offered.join(", "),
+                inner.builder_note(ROBOTS_TXT_TYPE, "robots.txt", written)
+            ),
+        }));
+    }
+    Ok(())
+}
+
 /// Resolves `[geo] module` against the registered geo modules.
 ///
 /// Returns the disabled module when the selector is unset or `none`, `None`
@@ -1693,6 +1764,15 @@ impl IntegrationRegistry {
                 claim_module_name(&mut claimed, "device", name, registration.integration_id)?;
                 inner.device_modules.push((name, module));
             }
+            if let Some((name, contributor)) = registration.robots_txt_contributor {
+                claim_module_name(
+                    &mut claimed,
+                    "robots.txt",
+                    name,
+                    registration.integration_id,
+                )?;
+                inner.robots_txt_contributors.push((name, contributor));
+            }
             if registration.js_disabled {
                 inner.disabled_js_ids.push(registration.integration_id);
             } else if registration.js_deferred {
@@ -1743,6 +1823,7 @@ impl IntegrationRegistry {
         inner.device_module = resolve_device_module(settings, &inner)?;
         inner.geo_module = geo_module;
         check_middleware(settings, &inner)?;
+        check_robots_txt_selection(settings, &inner)?;
 
         Ok(Self {
             inner: Arc::new(inner),
@@ -1780,6 +1861,23 @@ impl IntegrationRegistry {
         self.inner.device_module.clone()
     }
 
+    /// The robots.txt contributor `[robots-txt] modules` means by `written`,
+    /// with the name it was declared under, or `None` when nothing this
+    /// deployment runs supplies one under that name.
+    #[must_use]
+    pub fn robots_txt_contributor(
+        &self,
+        written: &str,
+    ) -> Option<(&'static str, Arc<dyn RobotsTxtContributor>)> {
+        let offered = &self.inner.robots_txt_contributors;
+        let names: Vec<&str> = offered.iter().map(|(name, _)| *name).collect();
+        let name = crate::module_name::resolve(ROBOTS_TXT_TYPE, written, &names)?;
+        offered
+            .iter()
+            .find(|(offered_name, _)| *offered_name == name)
+            .map(|(offered_name, contributor)| (*offered_name, Arc::clone(contributor)))
+    }
+
     /// The name of every middleware a running module supplies, in
     /// registration order.
     #[must_use]
@@ -2285,6 +2383,7 @@ impl IntegrationRegistry {
                 geo_modules: Vec::new(),
                 ec_modules: Vec::new(),
                 device_modules: Vec::new(),
+                robots_txt_contributors: Vec::new(),
                 builder_modules: Vec::new(),
                 geo_module: None,
                 ec_module: None,
@@ -2362,6 +2461,7 @@ impl IntegrationRegistry {
                 geo_modules: Vec::new(),
                 ec_modules: Vec::new(),
                 device_modules: Vec::new(),
+                robots_txt_contributors: Vec::new(),
                 builder_modules: Vec::new(),
                 geo_module: None,
                 ec_module: None,
diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs
index 75e55e8e41..190968ab51 100644
--- a/crates/trusted-server-core/src/lib.rs
+++ b/crates/trusted-server-core/src/lib.rs
@@ -69,6 +69,7 @@ pub mod publisher;
 pub mod redacted;
 pub mod request_signing;
 pub mod response_privacy;
+pub mod robots_txt;
 pub mod rsc_flight;
 pub(crate) mod s3_sigv4;
 pub mod secret_resolution;
diff --git a/crates/trusted-server-core/src/module_context.rs b/crates/trusted-server-core/src/module_context.rs
index 4b87f962b3..7a3595bfda 100644
--- a/crates/trusted-server-core/src/module_context.rs
+++ b/crates/trusted-server-core/src/module_context.rs
@@ -165,6 +165,15 @@ impl ResolvedRequest {
         }
     }
 
+    /// The same request with no query, for an answer that is held and served
+    /// to every caller, which must not depend on what one caller put in the
+    /// address.
+    #[must_use]
+    pub fn without_query(mut self) -> Self {
+        self.query.clear();
+        self
+    }
+
     /// The borrowed view a context carries.
     #[must_use]
     pub fn view(&self) -> ModuleRequest<'_> {
diff --git a/crates/trusted-server-core/src/robots_txt/contributor.rs b/crates/trusted-server-core/src/robots_txt/contributor.rs
new file mode 100644
index 0000000000..530a1ad82d
--- /dev/null
+++ b/crates/trusted-server-core/src/robots_txt/contributor.rs
@@ -0,0 +1,85 @@
+//! The extension point a module uses to add rules to `/robots.txt`.
+
+use core::time::Duration;
+
+use error_stack::Report;
+
+use super::Contribution;
+use crate::error::TrustedServerError;
+use crate::module_context::ModuleCall;
+use crate::permissions::PermissionSet;
+
+/// The `[robots-txt] modules` entry that refuses every crawler.
+///
+/// Core's own, and it names no vendor. Selecting it makes the file the refusal
+/// and nothing else, whatever else is selected, because a refusal cannot be
+/// loosened, and it puts `X-Robots-Tag: noindex, nofollow` on every response an
+/// adapter finalizes.
+pub const REFUSE_ALL: &str = "refuse_all";
+
+/// The `[robots-txt] modules` entry that allows every crawler everywhere.
+///
+/// Core's own, so permission is something the document says rather than
+/// something that follows from a lookup coming back empty.
+pub const ALLOW_ALL: &str = "allow_all";
+
+/// Something that adds rules to `/robots.txt`.
+///
+/// A module supplies one through
+/// [`IntegrationRegistrationBuilder::with_robots_txt_contributor`](crate::integrations::IntegrationRegistrationBuilder::with_robots_txt_contributor),
+/// and `[robots-txt] modules` selects it by the name it was declared under.
+/// Core holds what a contributor returns, assembles the file from every
+/// selected contributor in the order `modules` names them, and writes the
+/// bytes. The contributor never does, so it cannot produce a malformed file.
+///
+/// `Send + Sync` so the registry can share it between requests, while the
+/// future it returns is pinned to one thread, because the host SDKs produce
+/// `!Send` futures on wasm32.
+#[async_trait::async_trait(?Send)]
+pub trait RobotsTxtContributor: Send + Sync + core::fmt::Debug {
+    /// The longest one answer may be held before this contributor is asked
+    /// again.
+    ///
+    /// A ceiling, not a schedule. Each contributor's answer is held and
+    /// refreshed on its own, and the file is assembled from whatever each one
+    /// holds at the time, so two contributors never have to agree on anything.
+    fn refresh(&self) -> Duration;
+
+    /// A value that changes whenever this contributor's answer would, such as a
+    /// hash of its settings.
+    ///
+    /// It keys the held answers, so an answer given for different settings is
+    /// never served for these. Core hashes it into the key an answer is held
+    /// under, so nothing of it is written to the store.
+    fn fingerprint(&self) -> String;
+
+    /// The permissions this contributor declares, which decide whether a
+    /// gated value is passed to it. None by default.
+    fn required_permissions(&self) -> PermissionSet {
+        PermissionSet::none()
+    }
+
+    /// This contributor's rules.
+    ///
+    /// Core holds the answer for the publisher and serves it to every
+    /// crawler. So the call carries the request the answer was asked for,
+    /// being its method, path, host and scheme without its query, and the
+    /// request's services, and nothing about who asked. An implementation
+    /// hands its own function to [`ModuleCall::inject`], naming what it
+    /// needs. Its answer must be the same for every request the publisher
+    /// receives, because it is served for all of them.
+    ///
+    /// A contribution with no groups is an answer, and is served as one. An
+    /// error is a failure, and core serves this contributor's held copy
+    /// instead. With no held copy the whole file fails closed with `503`,
+    /// because leaving out a contributor's rules would allow by omission
+    /// whatever those rules refused.
+    ///
+    /// # Errors
+    ///
+    /// When this contributor cannot give its rules now.
+    async fn contribute(
+        &self,
+        call: ModuleCall<'_>,
+    ) -> Result<Contribution, Report<TrustedServerError>>;
+}
diff --git a/crates/trusted-server-core/src/robots_txt/document.rs b/crates/trusted-server-core/src/robots_txt/document.rs
new file mode 100644
index 0000000000..56d0665640
--- /dev/null
+++ b/crates/trusted-server-core/src/robots_txt/document.rs
@@ -0,0 +1,461 @@
+//! The file served at `/robots.txt`, held as a document rather than as text.
+//!
+//! Contributors hand core groups of records, and core writes every byte. So a
+//! contributor cannot produce a malformed file, and the rules that have to hold
+//! whatever a contributor says are applied here, in one place.
+//!
+//! 1. A refusal cannot be loosened. When `refuse_all` is selected the file is
+//!    the refusal and nothing else, see [`refusal`].
+//! 2. A path in `always_allow` stays open to every crawler a contribution
+//!    refuses the whole site to, see [`assemble`].
+//! 3. The publisher's own text and the `Sitemap` line go where the publisher
+//!    put them, around what the contributors gave.
+//!
+//! The record names in a contribution are written in one spelling,
+//! `User-Agent`, `Allow` and `Disallow`, and everything else is written back
+//! exactly as it was read. A contributor that reads a finished file in that
+//! spelling and hands it over as a [`Contribution`] therefore changes nothing in
+//! its bytes, which is what lets another implementation agree with this one byte
+//! for byte. The [`refusal`] is written whole rather than from contributions,
+//! and spells its first record `User-agent`.
+
+use serde::{Deserialize, Serialize};
+
+use super::RobotsTxtConfig;
+
+/// One record in a group, in the order it was given.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Record {
+    /// A path the group's crawlers may fetch.
+    Allow(String),
+    /// A path the group's crawlers may not fetch.
+    Disallow(String),
+    /// A line of explanation, without its leading `#`.
+    Comment(String),
+    /// Any other `name: value` record, kept where it was given, a Terms
+    /// Document Locator for example.
+    Other {
+        /// The record's name, as it was written.
+        name: String,
+        /// Its value.
+        value: String,
+    },
+}
+
+impl Record {
+    fn line(&self) -> String {
+        match self {
+            Self::Allow(path) => format!("Allow: {path}"),
+            Self::Disallow(path) => format!("Disallow: {path}"),
+            Self::Comment(text) => format!("#{text}"),
+            Self::Other { name, value } => format!("{name}: {value}"),
+        }
+    }
+
+    fn refuses_the_whole_site(&self) -> bool {
+        matches!(self, Self::Disallow(path) if path == "/")
+    }
+}
+
+/// The crawlers a group is for, and the records that apply to them.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Group {
+    /// Each written on its own `User-Agent` line. An empty name is kept as it
+    /// was given, because the file is written back as it was read.
+    pub user_agents: Vec<String>,
+    /// The group's records, in order.
+    pub records: Vec<Record>,
+}
+
+/// What one contributor adds to the file.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Contribution {
+    /// Records before the first group, such as a comment introducing the rules.
+    pub preamble: Vec<Record>,
+    /// The groups, in order.
+    pub groups: Vec<Group>,
+}
+
+impl Contribution {
+    /// A contribution read from `robots.txt` text, keeping every group, record
+    /// and comment in the order it came.
+    ///
+    /// A `User-Agent` line after a group's records starts the next group, and
+    /// consecutive `User-Agent` lines name the crawlers of one group, as RFC
+    /// 9309 section 2.1 describes. Blank lines and lines with no `:` carry
+    /// nothing and are passed over. A record before the first group is kept in
+    /// the preamble rather than dropped, so nothing given is lost.
+    #[must_use]
+    pub fn parse(text: &str) -> Self {
+        let mut contribution = Self::default();
+        let mut current: Option<Group> = None;
+        for raw in text.split('\n') {
+            let line = raw.trim();
+            if line.is_empty() {
+                continue;
+            }
+            if let Some(comment) = line.strip_prefix('#') {
+                push(
+                    &mut contribution,
+                    &mut current,
+                    Record::Comment(comment.to_owned()),
+                );
+                continue;
+            }
+            let Some((name, value)) = line.split_once(':') else {
+                continue;
+            };
+            let (name, value) = (name.trim(), value.trim().to_owned());
+            match name.to_ascii_lowercase().as_str() {
+                "user-agent" => {
+                    if current
+                        .as_ref()
+                        .is_some_and(|group| !group.records.is_empty())
+                        && let Some(finished) = current.take()
+                    {
+                        contribution.groups.push(finished);
+                    }
+                    current
+                        .get_or_insert_with(Group::default)
+                        .user_agents
+                        .push(value);
+                }
+                "allow" => push(&mut contribution, &mut current, Record::Allow(value)),
+                "disallow" => push(&mut contribution, &mut current, Record::Disallow(value)),
+                _ => push(
+                    &mut contribution,
+                    &mut current,
+                    Record::Other {
+                        name: name.to_owned(),
+                        value,
+                    },
+                ),
+            }
+        }
+        if let Some(group) = current {
+            contribution.groups.push(group);
+        }
+        contribution
+    }
+
+    /// Whether this contribution has anything to write.
+    #[must_use]
+    pub fn is_empty(&self) -> bool {
+        self.preamble.is_empty() && self.groups.is_empty()
+    }
+
+    /// The contribution as text, with a blank line between its parts and none
+    /// at either end.
+    fn write(&self) -> String {
+        let mut parts: Vec<String> = Vec::new();
+        if !self.preamble.is_empty() {
+            parts.push(
+                self.preamble
+                    .iter()
+                    .map(Record::line)
+                    .collect::<Vec<_>>()
+                    .join("\n"),
+            );
+        }
+        for group in &self.groups {
+            let mut lines: Vec<String> = group
+                .user_agents
+                .iter()
+                .map(|agent| format!("User-Agent: {agent}"))
+                .collect();
+            lines.extend(group.records.iter().map(Record::line));
+            parts.push(lines.join("\n"));
+        }
+        parts.join("\n\n")
+    }
+}
+
+fn push(contribution: &mut Contribution, current: &mut Option<Group>, record: Record) {
+    match current {
+        Some(group) => group.records.push(record),
+        None => contribution.preamble.push(record),
+    }
+}
+
+/// What `allow_all` contributes, which is every crawler allowed everywhere.
+///
+/// Permission the document says, rather than permission that follows from a
+/// lookup coming back empty. The two produce the same bytes, and only one of
+/// them was meant.
+#[must_use]
+pub fn allow_all() -> Contribution {
+    Contribution {
+        preamble: Vec::new(),
+        groups: vec![Group {
+            user_agents: vec!["*".to_owned()],
+            records: vec![Record::Allow("/".to_owned())],
+        }],
+    }
+}
+
+/// The file `refuse_all` serves, which refuses every crawler except for the
+/// paths in `always_allow`.
+///
+/// Nothing a contributor gives, and none of the publisher's own text, is
+/// added, because a refusal cannot be loosened. `Allow` beats `Disallow` on
+/// the longest match under RFC 9309 section 2.2.2, so each allowance lets that
+/// one path through.
+///
+/// A publisher selling advertising should allow `/ads.txt`, because the
+/// advertising platforms may ignore an `ads.txt` that robots.txt disallows,
+/// and a blanket `Disallow: /` is exactly that case. It is not added by
+/// default, because a domain with no `ads.txt` would be inviting crawlers to
+/// a path it does not have. The configuration guide quotes one platform's own
+/// statement of this.
+#[must_use]
+pub fn refusal(always_allow: &[String]) -> String {
+    let mut file = String::from("User-agent: *");
+    for path in always_allow {
+        file.push_str("\nAllow: ");
+        file.push_str(path);
+    }
+    file.push_str("\nDisallow: /\n");
+    file
+}
+
+/// The whole file, being the publisher's text above, each contribution in the
+/// order `modules` names it, the group that keeps `always_allow` open, the
+/// `Sitemap` line, and the publisher's text below.
+#[must_use]
+pub fn assemble(config: &RobotsTxtConfig, contributions: &[Contribution]) -> String {
+    let mut sections: Vec<String> = Vec::new();
+    if let Some(top) = config
+        .top_text
+        .as_deref()
+        .filter(|text| !text.trim().is_empty())
+    {
+        sections.push(top.trim_end().to_owned());
+    }
+    for contribution in contributions.iter().filter(|c| !c.is_empty()) {
+        sections.push(contribution.write());
+    }
+    if let Some(segment) = kept_open(contributions, &config.always_allow) {
+        sections.push(segment);
+    }
+    if let Some(sitemap) = &config.sitemap {
+        sections.push(format!("Sitemap: {sitemap}"));
+    }
+    if let Some(bottom) = config
+        .bottom_text
+        .as_deref()
+        .filter(|text| !text.trim().is_empty())
+    {
+        sections.push(bottom.trim_end().to_owned());
+    }
+    let mut file = sections.join("\n\n");
+    file.push('\n');
+    file
+}
+
+/// Every crawler the contributions refuse the whole site to, by the name on
+/// its `User-Agent` line, in order and without repeats. `*` is included, so
+/// the paths stay open to the crawlers no group names. A group with no name
+/// is passed over, because there is nothing to address a rule to.
+fn refused_entirely(contributions: &[Contribution]) -> Vec<&str> {
+    let mut names: Vec<&str> = Vec::new();
+    for group in contributions.iter().flat_map(|c| c.groups.iter()) {
+        if !group.records.iter().any(Record::refuses_the_whole_site) {
+            continue;
+        }
+        for name in &group.user_agents {
+            if !name.is_empty() && !names.contains(&name.as_str()) {
+                names.push(name);
+            }
+        }
+    }
+    names
+}
+
+/// A group of core's own that keeps each path in `always_allow` open to the
+/// crawlers the contributions refuse the whole site to, or `None` when there is
+/// nothing to keep open.
+///
+/// **The contributions are never edited.** RFC 9309 section 2.2.1 has a
+/// crawler combine every group that matches it, and section 2.2.2 gives the
+/// match with the most octets, so a second group naming the same crawlers with
+/// an `Allow` for the path beats `Disallow: /` for that path alone.
+///
+/// It goes below the contributions. A crawler that obeys only the first group
+/// it finds then still meets `Disallow: /` first, so the worst it can do is
+/// fail to read the one page. Written above, it would read the whole site.
+fn kept_open(contributions: &[Contribution], paths: &[String]) -> Option<String> {
+    let names = refused_entirely(contributions);
+    if names.is_empty() || paths.is_empty() {
+        return None;
+    }
+    let mut lines: Vec<String> = names
+        .iter()
+        .map(|name| format!("User-Agent: {name}"))
+        .collect();
+    lines.extend(paths.iter().map(|path| format!("Allow: {path}")));
+    Some(lines.join("\n"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn config() -> RobotsTxtConfig {
+        RobotsTxtConfig {
+            modules: vec!["a_contributor".to_owned()],
+            ..RobotsTxtConfig::refuse_all()
+        }
+    }
+
+    /// Text a contributor might read from somewhere else, with the shapes the
+    /// file has to survive: a record inside a group, crawlers sharing a group,
+    /// and a group with no name.
+    const GIVEN: &str = "User-Agent: ExampleBot\nDisallow: /\n\nUser-Agent: \nDisallow: /\n\n\
+                         User-Agent: OtherBot\nDisallow: /private/\n\nUser-Agent: ThirdBot\n\
+                         User-Agent: FourthBot\nDisallow: /\n\nUser-Agent: *\n\
+                         TDL: https://terms.example.com/socw/2.txt\nAllow: /";
+
+    #[test]
+    fn a_contribution_is_written_back_exactly_as_it_was_read() {
+        assert_eq!(Contribution::parse(GIVEN).write(), GIVEN);
+    }
+
+    #[test]
+    fn crawlers_named_together_share_one_group_and_a_record_keeps_its_place() {
+        let parsed = Contribution::parse(GIVEN);
+
+        assert_eq!(parsed.groups.len(), 5);
+        assert_eq!(parsed.groups[3].user_agents, vec!["ThirdBot", "FourthBot"]);
+        assert_eq!(
+            parsed.groups[4].records,
+            vec![
+                Record::Other {
+                    name: "TDL".to_owned(),
+                    value: "https://terms.example.com/socw/2.txt".to_owned(),
+                },
+                Record::Allow("/".to_owned()),
+            ]
+        );
+    }
+
+    #[test]
+    fn blank_lines_at_either_end_are_not_part_of_the_file() {
+        let parsed = Contribution::parse("\nUser-Agent: ExampleBot\nDisallow: /\n\n\n");
+
+        assert_eq!(parsed.write(), "User-Agent: ExampleBot\nDisallow: /");
+    }
+
+    #[test]
+    fn a_comment_and_a_record_before_any_group_are_kept() {
+        let parsed = Contribution::parse("# rules for crawlers\nUser-Agent: *\nAllow: /");
+
+        assert_eq!(
+            parsed.preamble,
+            vec![Record::Comment(" rules for crawlers".to_owned())]
+        );
+        assert_eq!(
+            parsed.write(),
+            "# rules for crawlers\n\nUser-Agent: *\nAllow: /"
+        );
+    }
+
+    #[test]
+    fn the_publishers_text_and_the_sitemap_go_around_the_contributions() {
+        let mut settings = config();
+        settings.top_text = Some("# top".to_owned());
+        settings.sitemap = Some("https://publisher.example.com/sitemap.xml".to_owned());
+        settings.bottom_text = Some("# bottom".to_owned());
+
+        assert_eq!(
+            assemble(
+                &settings,
+                &[Contribution::parse("User-agent: *\nAllow: /\n")]
+            ),
+            "# top\n\nUser-Agent: *\nAllow: /\n\n\
+             Sitemap: https://publisher.example.com/sitemap.xml\n\n# bottom\n",
+        );
+    }
+
+    /// The page a refused crawler is sent to stays open, by a group of core's
+    /// own below the contributions, which are not edited.
+    #[test]
+    fn an_always_allowed_path_is_kept_open_to_every_crawler_refused_the_site() {
+        let mut settings = config();
+        settings.always_allow = vec!["/ai-notice/".to_owned()];
+
+        let file = assemble(&settings, &[Contribution::parse(GIVEN)]);
+
+        assert!(
+            file.starts_with(GIVEN),
+            "the contribution is not edited: {file}"
+        );
+        assert!(
+            file.ends_with(
+                "\n\nUser-Agent: ExampleBot\nUser-Agent: ThirdBot\nUser-Agent: FourthBot\n\
+                 Allow: /ai-notice/\n"
+            ),
+            "the group naming every crawler refused the whole site goes last: {file}"
+        );
+    }
+
+    /// The path is kept open to `*` as it is to a crawler refused by name, so
+    /// the crawlers no group names can still read it.
+    #[test]
+    fn an_always_allowed_path_is_kept_open_when_every_crawler_is_refused() {
+        let mut settings = config();
+        settings.always_allow = vec!["/ai-notice/".to_owned()];
+
+        assert_eq!(
+            assemble(
+                &settings,
+                &[Contribution::parse("User-Agent: *\nDisallow: /")]
+            ),
+            "User-Agent: *\nDisallow: /\n\nUser-Agent: *\nAllow: /ai-notice/\n",
+            "a group for `*` below the contribution keeps the path open"
+        );
+    }
+
+    #[test]
+    fn nothing_is_kept_open_when_nothing_is_refused() {
+        let mut settings = config();
+        settings.always_allow = vec!["/ai-notice/".to_owned()];
+
+        assert_eq!(
+            assemble(&settings, &[allow_all()]),
+            "User-Agent: *\nAllow: /\n"
+        );
+    }
+
+    #[test]
+    fn contributions_appear_in_the_order_given() {
+        let first = Contribution::parse("User-Agent: FirstBot\nDisallow: /");
+        let second = Contribution::parse("User-Agent: SecondBot\nDisallow: /");
+
+        assert_eq!(
+            assemble(&config(), &[first, second]),
+            "User-Agent: FirstBot\nDisallow: /\n\nUser-Agent: SecondBot\nDisallow: /\n"
+        );
+    }
+
+    #[test]
+    fn an_empty_contribution_leaves_no_gap() {
+        assert_eq!(
+            assemble(&config(), &[Contribution::default(), allow_all()]),
+            "User-Agent: *\nAllow: /\n"
+        );
+    }
+
+    #[test]
+    fn refusing_with_no_allowances_refuses_everything() {
+        assert_eq!(refusal(&[]), "User-agent: *\nDisallow: /\n");
+    }
+
+    #[test]
+    fn each_allowance_is_written_in_order_ahead_of_the_refusal() {
+        assert_eq!(
+            refusal(&["/ads.txt".to_owned(), "/ai-notice/".to_owned()]),
+            "User-agent: *\nAllow: /ads.txt\nAllow: /ai-notice/\nDisallow: /\n"
+        );
+    }
+}
diff --git a/crates/trusted-server-core/src/robots_txt/holding.rs b/crates/trusted-server-core/src/robots_txt/holding.rs
new file mode 100644
index 0000000000..e81c78b56b
--- /dev/null
+++ b/crates/trusted-server-core/src/robots_txt/holding.rs
@@ -0,0 +1,650 @@
+//! Each contributor's answer, held in the key-value store between requests.
+//!
+//! A contributor is asked when nothing of its is held, or when what is held
+//! is older than the contributor's own refresh. Its answer is then stored and
+//! serves every request until the refresh is due again.
+//!
+//! What is held is kept far longer than any refresh, and is served past its
+//! age when the contributor fails, because yesterday's rules are a better
+//! answer than none. A `404` or an empty file reads to a crawler as permission
+//! to crawl everything, which is the opposite of what a publisher who refused
+//! anything asked for.
+//!
+//! A store that cannot give an entry a lifetime is written without one. An
+//! answer held for settings that have since changed then stays in that store,
+//! one small entry for each change, until it is deleted there.
+//!
+//! A deployment with no key-value store holds nothing. Every request for the
+//! file then asks each contributor, and a contributor that fails fails the
+//! file.
+
+use core::time::Duration;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use error_stack::Report;
+use serde::{Deserialize, Serialize};
+use sha2::{Digest as _, Sha256};
+
+use super::{Contribution, RobotsTxtContributor};
+use crate::error::TrustedServerError;
+use crate::module_context::{ModuleContext, ResolvedRequest};
+use crate::platform::{KvError, RuntimeServices};
+
+/// The start of every key an answer is held under, which nothing else that
+/// writes the store uses.
+const KEY_PREFIX: &str = "robots-txt:";
+
+/// How long an answer is kept, far past any refresh, so it is there to serve
+/// when the contributor cannot answer.
+const KEPT_LIFETIME: Duration = Duration::from_secs(30 * 24 * 3600);
+
+/// One contributor's answer as it is held.
+#[derive(Debug, Serialize, Deserialize)]
+struct Held {
+    /// When the contributor gave it, in seconds since the Unix epoch.
+    given_at: u64,
+    contribution: Contribution,
+}
+
+/// Who an answer is for and who gave it, which together name where it is
+/// held.
+pub(crate) struct Holder<'a> {
+    /// The publisher the file is for. One store may serve several, and one
+    /// publisher's rules are never another's.
+    pub(crate) publisher: &'a str,
+    /// The name `[robots-txt] modules` selects the contributor by.
+    pub(crate) id: &'static str,
+}
+
+/// The contribution of `contributor` from a held answer young enough to
+/// serve, and from the contributor otherwise, which is asked for `request`.
+///
+/// `now` is the time in seconds since the Unix epoch.
+///
+/// # Errors
+///
+/// When the contributor fails and no answer of its is held. A held answer is
+/// returned past its age rather than an error.
+pub(crate) async fn contribution_for(
+    holder: &Holder<'_>,
+    contributor: &Arc<dyn RobotsTxtContributor>,
+    request: &ResolvedRequest,
+    services: &RuntimeServices,
+    now: u64,
+) -> Result<Contribution, Report<TrustedServerError>> {
+    let key = held_key(holder, &contributor.fingerprint());
+    let held = held_answer(services, &key).await;
+    if let Some(held) = &held
+        && now.saturating_sub(held.given_at) < contributor.refresh().as_secs()
+    {
+        return Ok(held.contribution.clone());
+    }
+    let context = ModuleContext::new(request.view()).with_services(services);
+    let asked = contributor
+        .contribute(context.call(holder.id, contributor.required_permissions()))
+        .await;
+    match asked {
+        Ok(contribution) => {
+            let answer = Held {
+                given_at: now,
+                contribution,
+            };
+            hold_answer(services, &key, &answer).await;
+            Ok(answer.contribution)
+        }
+        Err(report) => match held {
+            Some(held) => {
+                log::warn!(
+                    "Serving the held robots.txt rules of `{}` past their age: {report:?}",
+                    holder.id
+                );
+                Ok(held.contribution)
+            }
+            None => Err(report.change_context(TrustedServerError::Integration {
+                integration: holder.id.to_owned(),
+                message: "the robots.txt contributor did not answer and none of its answers \
+                          is held"
+                    .to_owned(),
+            })),
+        },
+    }
+}
+
+/// The key one contributor's answer for one publisher and one fingerprint is
+/// held under. A hash, so that the fingerprint's contents, whatever they are,
+/// never appear in the store.
+fn held_key(holder: &Holder<'_>, fingerprint: &str) -> String {
+    let mut hasher = Sha256::new();
+    for part in [holder.publisher, holder.id, fingerprint] {
+        // Each part with its length, so that no two different triples run
+        // together into the same bytes.
+        hasher.update((part.len() as u64).to_be_bytes());
+        hasher.update(part.as_bytes());
+    }
+    let mut key = String::with_capacity(KEY_PREFIX.len() + 64);
+    key.push_str(KEY_PREFIX);
+    for byte in hasher.finalize() {
+        key.push_str(&format!("{byte:02x}"));
+    }
+    key
+}
+
+/// The answer held under `key`.
+///
+/// Nothing read, for whatever reason, is no held answer, and the contributor
+/// is asked.
+async fn held_answer(services: &RuntimeServices, key: &str) -> Option<Held> {
+    let bytes = match services.kv_store().get_bytes(key).await {
+        Ok(bytes) => bytes?,
+        Err(error) => {
+            log::debug!("No held robots.txt answer could be read: {error}");
+            return None;
+        }
+    };
+    serde_json::from_slice(&bytes)
+        .inspect_err(|error| log::debug!("The held robots.txt answer is unreadable: {error}"))
+        .ok()
+}
+
+/// Holds `answer` under `key`. An answer that cannot be held is still served,
+/// so a failure here is logged and nothing more.
+///
+/// A store that cannot give an entry a lifetime is written without one, so
+/// the answer is still there to serve when the contributor fails.
+async fn hold_answer(services: &RuntimeServices, key: &str, answer: &Held) {
+    let bytes = match serde_json::to_vec(answer) {
+        Ok(bytes) => Bytes::from(bytes),
+        Err(error) => {
+            log::debug!("The robots.txt answer cannot be held: {error}");
+            return;
+        }
+    };
+    let store = services.kv_store();
+    let written = match store
+        .put_bytes_with_ttl(key, bytes.clone(), KEPT_LIFETIME)
+        .await
+    {
+        Err(KvError::Unsupported { .. }) => store.put_bytes(key, bytes).await,
+        written => written,
+    };
+    if let Err(error) = written {
+        log::debug!("The robots.txt answer was not held: {error}");
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use std::collections::HashMap;
+    use std::sync::Mutex;
+
+    use edgezero_core::body::Body as EdgeBody;
+    use edgezero_core::key_value_store::{KvError, KvPage, KvStore as PlatformKvStore};
+
+    use super::*;
+    use crate::evidence::RequestInfo;
+    use crate::module_context::{ModuleCall, ModuleRequest};
+    use crate::platform::ClientInfo;
+    use crate::platform::test_support::noop_services;
+
+    /// The publisher the tests hold answers for.
+    pub(crate) const PUBLISHER: &str = "www.publisher.example";
+
+    /// A time to ask at, in seconds since the Unix epoch.
+    pub(crate) const NOW: u64 = 1_800_000_000;
+
+    /// A contributor whose answers a test decides, counting how often it is
+    /// asked.
+    #[derive(Debug)]
+    pub(crate) struct Scripted {
+        answers: Mutex<Vec<Result<Contribution, String>>>,
+        asked: Mutex<usize>,
+        pub(crate) refresh: Duration,
+        pub(crate) fingerprint: Mutex<String>,
+    }
+
+    impl Scripted {
+        pub(crate) fn new(answers: Vec<Result<Contribution, String>>) -> Self {
+            Self {
+                answers: Mutex::new(answers),
+                asked: Mutex::new(0),
+                refresh: Duration::from_secs(3600),
+                fingerprint: Mutex::new("first settings".to_owned()),
+            }
+        }
+
+        pub(crate) fn times_asked(&self) -> usize {
+            *self.asked.lock().expect("should lock the count")
+        }
+    }
+
+    #[async_trait::async_trait(?Send)]
+    impl RobotsTxtContributor for Scripted {
+        fn refresh(&self) -> Duration {
+            self.refresh
+        }
+
+        fn fingerprint(&self) -> String {
+            self.fingerprint.lock().expect("should lock").clone()
+        }
+
+        async fn contribute(
+            &self,
+            _call: ModuleCall<'_>,
+        ) -> Result<Contribution, Report<TrustedServerError>> {
+            *self.asked.lock().expect("should lock the count") += 1;
+            let next = self
+                .answers
+                .lock()
+                .expect("should lock the answers")
+                .remove(0);
+            next.map_err(|message| Report::new(TrustedServerError::Proxy { message }))
+        }
+    }
+
+    pub(crate) fn rules(text: &str) -> Contribution {
+        Contribution::parse(text)
+    }
+
+    /// A request for `/robots.txt` as a crawler makes it, with a query of its
+    /// own.
+    pub(crate) fn robots_request() -> http::Request<EdgeBody> {
+        http::Request::builder()
+            .uri("https://www.publisher.example/robots.txt?crawler=one")
+            .header(http::header::HOST, PUBLISHER)
+            .body(EdgeBody::empty())
+            .expect("should build the request")
+    }
+
+    /// The request the contributors are asked for, as the handler resolves it.
+    pub(crate) fn request() -> ResolvedRequest {
+        ResolvedRequest::of(&robots_request(), &ClientInfo::default()).without_query()
+    }
+
+    /// A contributor whose rules name what it was handed, so a test can read
+    /// back the request, whether the reader's evidence reached it, and
+    /// whether the services did.
+    #[derive(Debug)]
+    pub(crate) struct Echoing;
+
+    impl Echoing {
+        fn rules(
+            &self,
+            request: ModuleRequest<'_>,
+            evidence: Option<&dyn RequestInfo>,
+            services: Option<&RuntimeServices>,
+        ) -> Contribution {
+            rules(&format!(
+                "User-Agent: EchoBot\nDisallow: /{}{}?{}/{}/{}",
+                request.host(),
+                request.path(),
+                request.query(),
+                if evidence.is_some() {
+                    "evidence"
+                } else {
+                    "no-evidence"
+                },
+                if services.is_some() {
+                    "services"
+                } else {
+                    "no-services"
+                },
+            ))
+        }
+    }
+
+    #[async_trait::async_trait(?Send)]
+    impl RobotsTxtContributor for Echoing {
+        fn refresh(&self) -> Duration {
+            Duration::from_secs(60)
+        }
+
+        fn fingerprint(&self) -> String {
+            "echo".to_owned()
+        }
+
+        async fn contribute(
+            &self,
+            call: ModuleCall<'_>,
+        ) -> Result<Contribution, Report<TrustedServerError>> {
+            Ok(call.inject(self, Self::rules)?)
+        }
+    }
+
+    /// What [`Echoing`] answers for [`request`].
+    pub(crate) const ECHOED: &str =
+        "User-Agent: EchoBot\nDisallow: /www.publisher.example/robots.txt?/no-evidence/services";
+
+    /// A store held in memory for the life of one test, so that what one
+    /// request wrote is what the next one reads.
+    #[derive(Debug, Default)]
+    pub(crate) struct MemoryKvStore {
+        pub(crate) entries: Mutex<HashMap<String, Bytes>>,
+        /// The lifetime each entry was last written with.
+        pub(crate) lifetimes: Mutex<HashMap<String, Duration>>,
+        /// Whether a write with a lifetime is refused, as it is by a store
+        /// that has none to give.
+        pub(crate) refuses_lifetimes: bool,
+    }
+
+    #[async_trait::async_trait(?Send)]
+    impl PlatformKvStore for MemoryKvStore {
+        async fn get_bytes(&self, key: &str) -> Result<Option<Bytes>, KvError> {
+            Ok(self
+                .entries
+                .lock()
+                .expect("should lock the test store")
+                .get(key)
+                .cloned())
+        }
+
+        async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> {
+            self.entries
+                .lock()
+                .expect("should lock the test store")
+                .insert(key.to_owned(), value);
+            Ok(())
+        }
+
+        async fn put_bytes_with_ttl(
+            &self,
+            key: &str,
+            value: Bytes,
+            ttl: Duration,
+        ) -> Result<(), KvError> {
+            if self.refuses_lifetimes {
+                return Err(KvError::Unsupported {
+                    operation: "put_bytes_with_ttl".to_owned(),
+                });
+            }
+            self.lifetimes
+                .lock()
+                .expect("should lock the test store")
+                .insert(key.to_owned(), ttl);
+            self.put_bytes(key, value).await
+        }
+
+        async fn delete(&self, key: &str) -> Result<(), KvError> {
+            self.entries
+                .lock()
+                .expect("should lock the test store")
+                .remove(key);
+            Ok(())
+        }
+
+        async fn list_keys_page(
+            &self,
+            _prefix: &str,
+            _cursor: Option<&str>,
+            _limit: usize,
+        ) -> Result<KvPage, KvError> {
+            Ok(KvPage::default())
+        }
+    }
+
+    /// Services that hold answers in `store`.
+    pub(crate) fn services_with(store: Arc<MemoryKvStore>) -> RuntimeServices {
+        noop_services().with_kv_store(store)
+    }
+
+    fn contributor(scripted: &Arc<Scripted>) -> Arc<dyn RobotsTxtContributor> {
+        Arc::clone(scripted) as Arc<dyn RobotsTxtContributor>
+    }
+
+    fn holder(id: &'static str) -> Holder<'static> {
+        Holder {
+            publisher: PUBLISHER,
+            id,
+        }
+    }
+
+    async fn ask(
+        scripted: &Arc<Scripted>,
+        services: &RuntimeServices,
+        now: u64,
+    ) -> Result<Contribution, Report<TrustedServerError>> {
+        contribution_for(
+            &holder("a"),
+            &contributor(scripted),
+            &request(),
+            services,
+            now,
+        )
+        .await
+    }
+
+    /// The answer is shared by every crawler, so the contributor is handed
+    /// the request it was asked for and its services, and nothing about who
+    /// asked, not even the query.
+    #[tokio::test]
+    async fn the_contributor_is_asked_for_the_request_and_not_who_asked() {
+        let echoing: Arc<dyn RobotsTxtContributor> = Arc::new(Echoing);
+        let services = services_with(Arc::default());
+
+        let answer = contribution_for(&holder("echo"), &echoing, &request(), &services, NOW)
+            .await
+            .expect("should ask the contributor");
+
+        assert_eq!(answer, rules(ECHOED));
+    }
+
+    #[tokio::test]
+    async fn a_contributor_is_asked_once_and_its_answer_is_held_after_that() {
+        let scripted = Arc::new(Scripted::new(vec![Ok(rules("User-Agent: *\nAllow: /"))]));
+        let services = services_with(Arc::default());
+
+        let first = ask(&scripted, &services, NOW)
+            .await
+            .expect("should ask the contributor");
+        let second = ask(&scripted, &services, NOW + 3599)
+            .await
+            .expect("should serve the held answer");
+
+        assert_eq!(first, second);
+        assert_eq!(scripted.times_asked(), 1);
+    }
+
+    #[tokio::test]
+    async fn a_contributor_is_asked_again_when_its_refresh_is_due() {
+        let scripted = Arc::new(Scripted::new(vec![
+            Ok(rules("User-Agent: OneBot\nDisallow: /")),
+            Ok(rules("User-Agent: TwoBot\nDisallow: /")),
+        ]));
+        let services = services_with(Arc::default());
+
+        ask(&scripted, &services, NOW)
+            .await
+            .expect("should ask the contributor");
+        let later = ask(&scripted, &services, NOW + 3600)
+            .await
+            .expect("should ask the contributor again");
+
+        assert_eq!(later, rules("User-Agent: TwoBot\nDisallow: /"));
+        assert_eq!(scripted.times_asked(), 2);
+    }
+
+    /// Yesterday's rules are a better answer than none.
+    #[tokio::test]
+    async fn a_held_answer_is_served_past_its_age_when_the_contributor_fails() {
+        let scripted = Arc::new(Scripted::new(vec![
+            Ok(rules("User-Agent: OneBot\nDisallow: /")),
+            Err("cannot answer".to_owned()),
+        ]));
+        let services = services_with(Arc::default());
+
+        ask(&scripted, &services, NOW)
+            .await
+            .expect("should ask the contributor");
+        let later = ask(&scripted, &services, NOW + 7 * 24 * 3600)
+            .await
+            .expect("should serve the held answer past its age");
+
+        assert_eq!(later, rules("User-Agent: OneBot\nDisallow: /"));
+        assert_eq!(scripted.times_asked(), 2, "should have asked first");
+    }
+
+    #[tokio::test]
+    async fn with_nothing_held_a_failure_is_an_error() {
+        let scripted = Arc::new(Scripted::new(vec![Err("cannot answer".to_owned())]));
+        let services = services_with(Arc::default());
+
+        let error = ask(&scripted, &services, NOW)
+            .await
+            .expect_err("there is nothing to serve");
+
+        assert!(
+            format!("{error:?}").contains("cannot answer"),
+            "should keep what the contributor said: {error:?}"
+        );
+    }
+
+    /// An answer given for other settings is never served for these.
+    #[tokio::test]
+    async fn an_answer_is_held_for_the_fingerprint_it_was_given_for() {
+        let scripted = Arc::new(Scripted::new(vec![
+            Ok(rules("User-Agent: OneBot\nDisallow: /")),
+            Ok(rules("User-Agent: TwoBot\nDisallow: /")),
+        ]));
+        let services = services_with(Arc::default());
+
+        ask(&scripted, &services, NOW)
+            .await
+            .expect("should ask the contributor");
+        *scripted.fingerprint.lock().expect("should lock") = "second settings".to_owned();
+        let after = ask(&scripted, &services, NOW + 1)
+            .await
+            .expect("should ask for the new settings");
+
+        assert_eq!(after, rules("User-Agent: TwoBot\nDisallow: /"));
+    }
+
+    /// One store may serve several publishers, and one publisher's rules are
+    /// never another's.
+    #[tokio::test]
+    async fn an_answer_is_held_for_the_publisher_it_was_given_for() {
+        let scripted = Arc::new(Scripted::new(vec![
+            Ok(rules("User-Agent: OneBot\nDisallow: /")),
+            Ok(rules("User-Agent: TwoBot\nDisallow: /")),
+        ]));
+        let services = services_with(Arc::default());
+        let other = Holder {
+            publisher: "www.another.example",
+            id: "a",
+        };
+
+        ask(&scripted, &services, NOW)
+            .await
+            .expect("should ask the contributor");
+        let for_other = contribution_for(
+            &other,
+            &contributor(&scripted),
+            &request(),
+            &services,
+            NOW + 1,
+        )
+        .await
+        .expect("should ask for the other publisher");
+
+        assert_eq!(for_other, rules("User-Agent: TwoBot\nDisallow: /"));
+    }
+
+    #[test]
+    fn a_key_holds_nothing_of_what_it_was_made_from() {
+        let key = held_key(&holder("a_contributor"), "a secret an implementer left in");
+
+        assert!(key.starts_with(KEY_PREFIX), "{key}");
+        assert_eq!(key.len(), KEY_PREFIX.len() + 64, "{key}");
+        for part in [PUBLISHER, "a_contributor", "secret"] {
+            assert!(!key.contains(part), "{key} should not hold {part}");
+        }
+    }
+
+    #[test]
+    fn parts_that_run_together_do_not_share_a_key() {
+        let one = held_key(
+            &Holder {
+                publisher: "ab",
+                id: "c",
+            },
+            "d",
+        );
+        let other = held_key(
+            &Holder {
+                publisher: "a",
+                id: "bc",
+            },
+            "d",
+        );
+
+        assert_ne!(one, other);
+    }
+
+    #[tokio::test]
+    async fn an_answer_is_kept_far_longer_than_any_refresh() {
+        let scripted = Arc::new(Scripted::new(vec![Ok(rules("User-Agent: *\nAllow: /"))]));
+        let store = Arc::new(MemoryKvStore::default());
+        let services = services_with(Arc::clone(&store));
+
+        ask(&scripted, &services, NOW)
+            .await
+            .expect("should ask the contributor");
+
+        let lifetimes = store.lifetimes.lock().expect("should lock the test store");
+        assert_eq!(
+            lifetimes.values().copied().collect::<Vec<_>>(),
+            vec![KEPT_LIFETIME]
+        );
+    }
+
+    /// A store with no lifetime to give still holds the answer, so it is
+    /// there to serve when the contributor fails.
+    #[tokio::test]
+    async fn a_store_without_lifetimes_still_holds_the_answer() {
+        let scripted = Arc::new(Scripted::new(vec![
+            Ok(rules("User-Agent: OneBot\nDisallow: /")),
+            Err("cannot answer".to_owned()),
+        ]));
+        let store = Arc::new(MemoryKvStore {
+            refuses_lifetimes: true,
+            ..MemoryKvStore::default()
+        });
+        let services = services_with(Arc::clone(&store));
+
+        ask(&scripted, &services, NOW)
+            .await
+            .expect("should ask the contributor");
+        let later = ask(&scripted, &services, NOW + 3600)
+            .await
+            .expect("should serve the held answer");
+
+        assert_eq!(later, rules("User-Agent: OneBot\nDisallow: /"));
+        assert!(
+            store
+                .lifetimes
+                .lock()
+                .expect("should lock the test store")
+                .is_empty(),
+            "the entry was written without a lifetime"
+        );
+    }
+
+    /// A deployment with no key-value store still serves the file, by asking
+    /// every time.
+    #[tokio::test]
+    async fn with_no_store_the_contributor_is_asked_every_time() {
+        let scripted = Arc::new(Scripted::new(vec![
+            Ok(rules("User-Agent: *\nAllow: /")),
+            Ok(rules("User-Agent: *\nAllow: /")),
+        ]));
+        let services = noop_services();
+
+        ask(&scripted, &services, NOW)
+            .await
+            .expect("should ask the contributor");
+        ask(&scripted, &services, NOW + 1)
+            .await
+            .expect("should ask the contributor again");
+
+        assert_eq!(scripted.times_asked(), 2);
+    }
+}
diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs
new file mode 100644
index 0000000000..86d0f3b77b
--- /dev/null
+++ b/crates/trusted-server-core/src/robots_txt/mod.rs
@@ -0,0 +1,827 @@
+//! A publisher's `robots.txt`, assembled from the rules the selected
+//! contributors give.
+//!
+//! `[robots-txt] modules` names, in order, what makes the file. Two names are
+//! core's own and need nothing else: [`REFUSE_ALL`] refuses every crawler, and
+//! [`ALLOW_ALL`] allows every crawler everywhere. Any other name is the name a
+//! module declared a [`RobotsTxtContributor`] under. Core never names a
+//! module here and knows nothing of what any contributor asks or of whom.
+//!
+//! # Contributors add rules, core writes the file
+//!
+//! A contributor returns a [`Contribution`], being groups of records, and core
+//! writes every byte in [`document`]. That is what lets these hold whatever a
+//! contributor says.
+//!
+//! 1. A refusal cannot be loosened. With `refuse_all` selected the file is the
+//!    refusal and nothing else.
+//! 2. A path in `always_allow` stays open to every crawler a contribution
+//!    refuses the whole site to, `*` included. A narrower rule is kept as
+//!    given, so `Disallow: /ads` still closes off `/ads.txt`.
+//! 3. The `Sitemap` line, the publisher's own text, and the `X-Robots-Tag`
+//!    header on every page agree with the file served.
+//! 4. Several contributors combine in the order `modules` gives, and none of
+//!    them owns the file.
+//! 5. A contributor never writes bytes, so it cannot produce a malformed file.
+//! 6. A contributor that fails with no held answer fails the whole file closed
+//!    with `503`, because leaving out a contributor's rules would allow by
+//!    omission whatever those rules refused. A `404` or an empty file reads to
+//!    a crawler as permission to crawl everything.
+//!
+//! # Settings
+//!
+//! `[robots-txt]` holds `modules` and the document's own settings, being
+//! `always_allow`, `sitemap`, `top_text` and `bottom_text`. Settings belonging
+//! to one contributor go in its own block, `[robots-txt.<name>]`, which that
+//! contributor reads and core does not.
+
+mod contributor;
+pub mod document;
+mod holding;
+
+use std::collections::BTreeMap;
+use std::sync::Arc;
+
+use edgezero_core::body::Body as EdgeBody;
+use error_stack::Report;
+use http::header::{self, HeaderValue};
+use http::{Method, Request, Response, StatusCode};
+use serde::{Deserialize, Deserializer, Serialize};
+use validator::{Validate, ValidationError};
+
+pub use contributor::{ALLOW_ALL, REFUSE_ALL, RobotsTxtContributor};
+pub use document::{Contribution, Group, Record};
+
+use crate::error::TrustedServerError;
+use crate::integrations::IntegrationRegistry;
+use crate::module_context::ResolvedRequest;
+use crate::platform::RuntimeServices;
+use crate::settings::Settings;
+
+/// The path this module answers.
+pub const ROBOTS_TXT_PATH: &str = "/robots.txt";
+
+/// How long a browser or an edge cache may hold the answer.
+const CLIENT_CACHE_SECONDS: u32 = 3600;
+
+/// How long a crawler is asked to wait when there is no file to give it.
+const RETRY_AFTER_SECONDS: u32 = 600;
+
+/// The `X-Robots-Tag` a response carries when `refuse_all` is selected.
+///
+/// A robots.txt asks a crawler not to fetch. This tells one that has already
+/// fetched not to index, and it is the only one of the two that works on a page
+/// a crawler learned about from somewhere else, or reached before reading
+/// robots.txt, or chose to fetch anyway. They do different jobs and a site that
+/// is not ready to be seen needs both.
+///
+/// Set by the same selection so the two cannot drift apart.
+///
+/// `nofollow` as well as `noindex`, so a crawler does not walk on from a page
+/// it should not have indexed into the rest of a site that is not ready either.
+pub const REFUSE_ALL_ROBOTS_TAG: &str = "noindex, nofollow";
+
+/// The type folder of every robots.txt crate, which a name written in
+/// `[robots-txt] modules` may leave off.
+pub const MODULE_TYPE: &str = "robots-txt";
+
+/// Settings for serving `robots.txt`.
+///
+/// Mapped from the `[robots-txt]` TOML section. With the section absent the
+/// publisher keeps their own file and `/robots.txt` reaches their origin like
+/// any other path.
+#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
+pub struct RobotsTxtConfig {
+    /// What makes the file, in order. One name or a list, so
+    /// `modules = "refuse_all"` and `modules = ["refuse_all"]` are the same.
+    ///
+    /// Required, and refused at load when absent or empty. There is no default,
+    /// because a default would decide for a publisher what crawlers may do.
+    #[serde(default, deserialize_with = "one_or_many")]
+    pub modules: Vec<String>,
+
+    /// Paths kept open to every crawler refused the whole site, the page a
+    /// refused crawler is sent to for example. A narrower rule, such as
+    /// `Disallow: /ads`, still closes off `/ads.txt`. `/robots.txt` needs no
+    /// entry, because RFC 9309 section 2.2.2 says it is implicitly allowed.
+    ///
+    /// A publisher who sells advertising almost certainly wants `/ads.txt`
+    /// here, for the reason given on [`document::refusal`].
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    #[validate(custom(function = validate_always_allow))]
+    pub always_allow: Vec<String>,
+
+    /// The site's sitemap, added as a `Sitemap:` line.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    #[validate(url)]
+    pub sitemap: Option<String>,
+
+    /// The publisher's own text, placed above the contributors' rules.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub top_text: Option<String>,
+
+    /// The publisher's own text, placed below the contributors' rules.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub bottom_text: Option<String>,
+
+    /// Each contributor's own settings, `[robots-txt.<name>]`, keyed by its
+    /// name.
+    ///
+    /// Read by that contributor and by nothing in core, which only checks that
+    /// each is a block and belongs to a contributor `modules` names.
+    #[serde(flatten)]
+    pub contributors: BTreeMap<String, serde_json::Value>,
+}
+
+fn one_or_many<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    #[derive(Deserialize)]
+    #[serde(untagged)]
+    enum OneOrMany {
+        One(String),
+        Many(Vec<String>),
+    }
+    Ok(match OneOrMany::deserialize(deserializer)? {
+        OneOrMany::One(name) => vec![name],
+        OneOrMany::Many(names) => names,
+    })
+}
+
+fn validate_always_allow(paths: &[String]) -> Result<(), ValidationError> {
+    // A path is written into the file as it stands, so it must be a path and
+    // must not be able to start a second line.
+    if paths
+        .iter()
+        .any(|path| !path.starts_with('/') || path.chars().any(char::is_whitespace))
+    {
+        return Err(ValidationError::new("robots_txt_always_allow_not_a_path"));
+    }
+    Ok(())
+}
+
+impl RobotsTxtConfig {
+    /// The configuration that refuses every crawler with no exception.
+    ///
+    /// The starting point for a publisher who wants the refusal with
+    /// allowances of their own. Needs no contributor, no key and no network.
+    #[must_use]
+    pub fn refuse_all() -> Self {
+        Self {
+            modules: vec![REFUSE_ALL.to_owned()],
+            always_allow: Vec::new(),
+            sitemap: None,
+            top_text: None,
+            bottom_text: None,
+            contributors: BTreeMap::new(),
+        }
+    }
+
+    /// Whether `modules` names `id`, by its full name or by its short form,
+    /// whichever of the two `id` and the name in `modules` are written in.
+    #[must_use]
+    pub fn selects(&self, id: &str) -> bool {
+        self.modules.iter().any(|selected| {
+            crate::module_name::resolve(MODULE_TYPE, selected, &[id]).is_some()
+                || crate::module_name::short_form(MODULE_TYPE, selected) == id
+        })
+    }
+
+    /// Whether the file is the refusal, which it is whenever `refuse_all` is
+    /// selected, whatever else is.
+    #[must_use]
+    pub fn refuses_all(&self) -> bool {
+        self.selects(REFUSE_ALL)
+    }
+
+    /// The `X-Robots-Tag` every response should carry, or `None` when the
+    /// publisher's crawler policy places no blanket rule on pages.
+    #[must_use]
+    pub fn response_robots_tag(&self) -> Option<&'static str> {
+        self.refuses_all().then_some(REFUSE_ALL_ROBOTS_TAG)
+    }
+
+    /// The settings block of the contributor named `id`, when there is one.
+    #[must_use]
+    pub fn contributor_settings(&self, id: &str) -> Option<&serde_json::Value> {
+        self.contributors.get(id).or_else(|| {
+            self.contributors
+                .get(crate::module_name::short_form(MODULE_TYPE, id))
+        })
+    }
+
+    /// Checks what the type cannot.
+    ///
+    /// # Errors
+    ///
+    /// When `modules` names nothing or names one thing twice, when a key is
+    /// not a setting, and when a contributor's block belongs to one `modules`
+    /// does not name.
+    pub fn prepare_runtime(&self) -> Result<(), Report<TrustedServerError>> {
+        if self.modules.is_empty() {
+            return Err(Report::new(configuration(
+                "[robots-txt] names no module. Name what makes the file, in order: \
+                 `refuse_all` to refuse every crawler, `allow_all` to allow every crawler, \
+                 or the name of each module whose rules make the file. There is no default, \
+                 because a default would decide for the publisher what crawlers may do."
+                    .to_owned(),
+            )));
+        }
+        // Compared on the short form, so the full name and the short form of
+        // one module count as naming it twice.
+        for (index, name) in self.modules.iter().enumerate() {
+            let short_name = crate::module_name::short_form(MODULE_TYPE, name);
+            if self.modules[..index]
+                .iter()
+                .any(|earlier| crate::module_name::short_form(MODULE_TYPE, earlier) == short_name)
+            {
+                return Err(Report::new(configuration(format!(
+                    "[robots-txt] modules names `{name}` twice."
+                ))));
+            }
+        }
+        for (key, value) in &self.contributors {
+            if !value.is_object() {
+                return Err(Report::new(configuration(format!(
+                    "robots-txt.{key} is not a setting. [robots-txt] holds modules, \
+                     always_allow, sitemap, top_text, bottom_text, and a block for each \
+                     contributor that has settings of its own."
+                ))));
+            }
+            if !self.selects(key) {
+                return Err(Report::new(configuration(format!(
+                    "[robots-txt.{key}] configures a contributor that `modules` does not \
+                     name, so nothing would read it. Add `{key}` to modules or remove the \
+                     block."
+                ))));
+            }
+        }
+        Ok(())
+    }
+}
+
+fn configuration(message: String) -> TrustedServerError {
+    TrustedServerError::Configuration { message }
+}
+
+/// Applies the publisher's blanket crawler rule to an outgoing response.
+///
+/// Called once where an adapter finalizes a response, rather than on each
+/// route, so a route added later cannot be the one page a crawler is allowed
+/// to index. A site taken out of search has to be taken out of search
+/// entirely.
+///
+/// Replaces rather than appends. Selecting `refuse_all` means the whole site is
+/// not to be indexed, so a tag an origin page set for its own reasons must not
+/// be able to weaken it, and two tags on one response is how a crawler is given
+/// a choice.
+///
+/// Does nothing unless `refuse_all` is selected, because a file built from
+/// contributors already says what each crawler may do.
+pub fn apply_response_robots_tag(settings: &Settings, response: &mut Response<EdgeBody>) {
+    if settings
+        .robots_txt
+        .as_ref()
+        .and_then(RobotsTxtConfig::response_robots_tag)
+        .is_none()
+    {
+        return;
+    }
+    response.headers_mut().insert(
+        header::HeaderName::from_static("x-robots-tag"),
+        HeaderValue::from_static(REFUSE_ALL_ROBOTS_TAG),
+    );
+}
+
+/// Answers `GET` and `HEAD /robots.txt` for a publisher whose settings carry
+/// a `[robots-txt]` section. A `HEAD` is answered with the headers of the
+/// `GET` and no body.
+///
+/// # Errors
+///
+/// When the settings carry no `[robots-txt]` section. An adapter registers
+/// this route only when the section is present, so that is a wiring fault and
+/// not something a request can cause.
+pub async fn handle_robots_txt(
+    settings: &Settings,
+    services: &RuntimeServices,
+    registry: &IntegrationRegistry,
+    req: Request<EdgeBody>,
+) -> Result<Response<EdgeBody>, Report<TrustedServerError>> {
+    let response = answer(settings, services, registry, &req).await?;
+    Ok(if req.method() == Method::HEAD {
+        response.map(|_| EdgeBody::empty())
+    } else {
+        response
+    })
+}
+
+async fn answer(
+    settings: &Settings,
+    services: &RuntimeServices,
+    registry: &IntegrationRegistry,
+    req: &Request<EdgeBody>,
+) -> Result<Response<EdgeBody>, Report<TrustedServerError>> {
+    let config = settings.robots_txt.as_ref().ok_or_else(|| {
+        Report::new(configuration(
+            "the robots.txt route is registered but there is no [robots-txt] section".to_owned(),
+        ))
+    })?;
+    // A refusal answers without asking anything, before any contributor,
+    // held answer or network is involved. That is the point of it, because a
+    // site that is not ready to be crawled can then say so with nothing
+    // provisioned at all.
+    if config.refuses_all() {
+        return Ok(respond(Ok(document::refusal(&config.always_allow))));
+    }
+    let mut selected: Vec<Selected> = Vec::new();
+    for id in &config.modules {
+        if id == ALLOW_ALL {
+            selected.push(None);
+            continue;
+        }
+        let Some(contributor) = registry.robots_txt_contributor(id) else {
+            // The registry refuses a selection nothing supplies when it is
+            // built, so this is a wiring fault, and it still fails closed.
+            return Ok(respond(Err(Report::new(configuration(format!(
+                "[robots-txt] modules names `{id}`, which no module in this deployment supplies"
+            ))))));
+        };
+        selected.push(Some(contributor));
+    }
+    // The answers are held and served to every crawler, so the contributors
+    // are handed the request with nothing one crawler put in the address.
+    let request = ResolvedRequest::of(req, services.client_info()).without_query();
+    Ok(respond(
+        file_for(
+            config,
+            &selected,
+            &request,
+            services,
+            &settings.publisher.domain,
+            crate::ec::current_timestamp(),
+        )
+        .await,
+    ))
+}
+
+/// One entry of `modules` as the handler resolved it. `None` is `allow_all`,
+/// and anything else is a contributor with the name it was declared under.
+type Selected = Option<(&'static str, Arc<dyn RobotsTxtContributor>)>;
+
+/// The file for `config`, from the contributions of `selected` in order,
+/// each asked for `request` and held for `publisher`.
+async fn file_for(
+    config: &RobotsTxtConfig,
+    selected: &[Selected],
+    request: &ResolvedRequest,
+    services: &RuntimeServices,
+    publisher: &str,
+    now: u64,
+) -> Result<String, Report<TrustedServerError>> {
+    let mut contributions = Vec::with_capacity(selected.len());
+    for entry in selected {
+        contributions.push(match entry {
+            None => document::allow_all(),
+            Some((id, contributor)) => {
+                let holder = holding::Holder { publisher, id };
+                holding::contribution_for(&holder, contributor, request, services, now).await?
+            }
+        });
+    }
+    Ok(document::assemble(config, &contributions))
+}
+
+fn respond(file: Result<String, Report<TrustedServerError>>) -> Response<EdgeBody> {
+    match file {
+        Ok(file) => Response::builder()
+            .status(StatusCode::OK)
+            .header(
+                header::CONTENT_TYPE,
+                HeaderValue::from_static("text/plain; charset=utf-8"),
+            )
+            .header(
+                header::CACHE_CONTROL,
+                format!("public, max-age={CLIENT_CACHE_SECONDS}"),
+            )
+            .body(EdgeBody::from(file.into_bytes()))
+            .expect("should build the robots.txt response"),
+        Err(report) => {
+            log::error!("Could not assemble robots.txt: {report:?}");
+            Response::builder()
+                .status(StatusCode::SERVICE_UNAVAILABLE)
+                .header(
+                    header::CONTENT_TYPE,
+                    HeaderValue::from_static("text/plain; charset=utf-8"),
+                )
+                .header(header::RETRY_AFTER, RETRY_AFTER_SECONDS.to_string())
+                .header(header::CACHE_CONTROL, HeaderValue::from_static("no-store"))
+                .body(EdgeBody::from(
+                    b"robots.txt is temporarily unavailable\n".to_vec(),
+                ))
+                .expect("should build the robots.txt failure response")
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use super::holding::tests::{
+        ECHOED, Echoing, MemoryKvStore, NOW, PUBLISHER, Scripted, request, robots_request, rules,
+        services_with,
+    };
+    use super::*;
+    use crate::integrations::{IntegrationBuilder, IntegrationRegistration};
+
+    fn parsed(document: serde_json::Value) -> RobotsTxtConfig {
+        serde_json::from_value(document).expect("should read the robots.txt settings")
+    }
+
+    fn settings() -> Settings {
+        let mut settings = crate::test_support::tests::create_test_settings();
+        settings.publisher.domain = PUBLISHER.to_owned();
+        settings
+    }
+
+    #[test]
+    fn a_single_module_reads_as_a_list_of_one() {
+        let config = parsed(serde_json::json!({ "modules": "refuse_all" }));
+
+        assert_eq!(config.modules, vec!["refuse_all"]);
+        assert!(config.refuses_all());
+        config
+            .prepare_runtime()
+            .expect("a single module written as a string must load");
+    }
+
+    #[test]
+    fn a_list_of_modules_keeps_its_order() {
+        let config = parsed(serde_json::json!({
+            "modules": ["a_module", "allow_all"],
+            "a_module": { "anything": "the module reads" },
+        }));
+
+        assert_eq!(config.modules, vec!["a_module", "allow_all"]);
+        config.prepare_runtime().expect("should load");
+    }
+
+    #[test]
+    fn no_module_is_refused_rather_than_defaulted() {
+        let error = parsed(serde_json::json!({ "sitemap": "https://example.com/sitemap.xml" }))
+            .prepare_runtime()
+            .expect_err("there is no default");
+        let message = format!("{error:?}");
+
+        assert!(message.contains("refuse_all"), "{message}");
+        assert!(message.contains("allow_all"), "{message}");
+    }
+
+    #[test]
+    fn an_empty_list_is_refused_the_same_way() {
+        assert!(
+            parsed(serde_json::json!({ "modules": [] }))
+                .prepare_runtime()
+                .is_err()
+        );
+    }
+
+    #[test]
+    fn a_name_given_twice_is_refused() {
+        assert!(
+            parsed(serde_json::json!({ "modules": ["allow_all", "allow_all"] }))
+                .prepare_runtime()
+                .is_err()
+        );
+    }
+
+    /// A section may write a contributor's full name or leave the type folder
+    /// off, so both spellings select the block, and naming both is naming the
+    /// contributor twice.
+    #[test]
+    fn a_contributor_named_in_full_selects_its_block() {
+        parsed(serde_json::json!({
+            "modules": "robots-txt.a_module",
+            "a_module": { "anything": "the module reads" },
+        }))
+        .prepare_runtime()
+        .expect("the full name should select the block written at the short name");
+
+        let error = parsed(serde_json::json!({
+            "modules": ["a_module", "robots-txt.a_module"],
+            "a_module": { "anything": "the module reads" },
+        }))
+        .prepare_runtime()
+        .expect_err("one contributor named in two spellings is named twice");
+        assert!(format!("{error:?}").contains("twice"), "{error:?}");
+    }
+
+    #[test]
+    fn a_key_that_is_not_a_setting_is_refused() {
+        let message = format!(
+            "{:?}",
+            parsed(serde_json::json!({ "modules": "allow_all", "sitmap": "a typo" }))
+                .prepare_runtime()
+                .expect_err("a misspelled setting must not load")
+        );
+
+        assert!(message.contains("is not a setting"), "{message}");
+    }
+
+    #[test]
+    fn a_block_for_a_contributor_not_selected_is_refused() {
+        let message = format!(
+            "{:?}",
+            parsed(serde_json::json!({ "modules": "allow_all", "a_module": {} }))
+                .prepare_runtime()
+                .expect_err("nothing would read the block")
+        );
+
+        assert!(message.contains("does not name"), "{message}");
+    }
+
+    #[test]
+    fn a_path_kept_open_must_be_a_path_on_one_line() {
+        for path in ["ads.txt", "/ads.txt\nDisallow: /", "/a path"] {
+            let config = parsed(serde_json::json!({
+                "modules": "refuse_all",
+                "always_allow": [path],
+            }));
+
+            assert!(config.validate().is_err(), "should refuse {path:?}");
+        }
+        parsed(serde_json::json!({ "modules": "refuse_all", "always_allow": ["/ads.txt"] }))
+            .validate()
+            .expect("should accept a path");
+    }
+
+    #[test]
+    fn refusing_sets_the_response_header_and_a_contributor_does_not() {
+        assert_eq!(
+            RobotsTxtConfig::refuse_all().response_robots_tag(),
+            Some(REFUSE_ALL_ROBOTS_TAG)
+        );
+        assert_eq!(
+            parsed(serde_json::json!({ "modules": "allow_all" })).response_robots_tag(),
+            None
+        );
+    }
+
+    #[test]
+    fn the_header_replaces_one_the_origin_set() {
+        let mut settings = settings();
+        settings.robots_txt = Some(RobotsTxtConfig::refuse_all());
+        let mut response = Response::builder()
+            .header("x-robots-tag", "all")
+            .body(EdgeBody::empty())
+            .expect("should build a response");
+
+        apply_response_robots_tag(&settings, &mut response);
+
+        let tags: Vec<_> = response.headers().get_all("x-robots-tag").iter().collect();
+        assert_eq!(tags, vec![REFUSE_ALL_ROBOTS_TAG]);
+    }
+
+    #[test]
+    fn a_response_is_left_alone_when_nothing_refuses_every_crawler() {
+        let mut allowing = settings();
+        allowing.robots_txt = Some(parsed(serde_json::json!({ "modules": "allow_all" })));
+        let mut without_a_section = settings();
+        without_a_section.robots_txt = None;
+
+        for settings in [allowing, without_a_section] {
+            let mut response = Response::builder()
+                .header("x-robots-tag", "noarchive")
+                .body(EdgeBody::empty())
+                .expect("should build a response");
+
+            apply_response_robots_tag(&settings, &mut response);
+
+            let tags: Vec<_> = response.headers().get_all("x-robots-tag").iter().collect();
+            assert_eq!(tags, vec!["noarchive"]);
+        }
+    }
+
+    #[tokio::test]
+    async fn contributions_are_assembled_in_the_order_selected() {
+        let first: Arc<dyn RobotsTxtContributor> = Arc::new(Scripted::new(vec![Ok(rules(
+            "User-Agent: FirstBot\nDisallow: /",
+        ))]));
+        let config = parsed(serde_json::json!({
+            "modules": ["first", "allow_all"],
+            "sitemap": "https://publisher.example.com/sitemap.xml",
+        }));
+
+        let file = file_for(
+            &config,
+            &[Some(("first", first)), None],
+            &request(),
+            &services_with(Arc::default()),
+            PUBLISHER,
+            NOW,
+        )
+        .await
+        .expect("should assemble the file");
+
+        assert_eq!(
+            file,
+            "User-Agent: FirstBot\nDisallow: /\n\nUser-Agent: *\nAllow: /\n\n\
+             Sitemap: https://publisher.example.com/sitemap.xml\n"
+        );
+    }
+
+    /// Leaving out the rules of a contributor that failed would allow by
+    /// omission what those rules refused, so the whole file fails closed.
+    #[tokio::test]
+    async fn with_nothing_held_a_failure_is_service_unavailable() {
+        let failing: Arc<dyn RobotsTxtContributor> =
+            Arc::new(Scripted::new(vec![Err("cannot answer".to_owned())]));
+        let config = parsed(serde_json::json!({ "modules": ["failing", "allow_all"] }));
+
+        let response = respond(
+            file_for(
+                &config,
+                &[Some(("failing", failing)), None],
+                &request(),
+                &services_with(Arc::default()),
+                PUBLISHER,
+                NOW,
+            )
+            .await,
+        );
+
+        assert_eq!(
+            response.status(),
+            StatusCode::SERVICE_UNAVAILABLE,
+            "a crawler reads an empty or allow everything file as permission"
+        );
+        assert!(response.headers().contains_key(header::RETRY_AFTER));
+        assert_eq!(
+            response.headers().get(header::CACHE_CONTROL),
+            Some(&HeaderValue::from_static("no-store")),
+            "a failure must not be held in place of the file"
+        );
+    }
+
+    /// The refusal asks nothing of anything, and dominates whatever else is
+    /// selected, because a refusal cannot be loosened.
+    #[tokio::test]
+    async fn refusing_every_crawler_asks_no_contributor_and_reads_no_store() {
+        let mut settings = settings();
+        let mut refusing = parsed(serde_json::json!({ "modules": ["allow_all", "refuse_all"] }));
+        refusing.always_allow = vec!["/ads.txt".to_owned()];
+        settings.robots_txt = Some(refusing);
+        let store = Arc::new(MemoryKvStore::default());
+        let registry = IntegrationRegistry::new(&settings).expect("should build the registry");
+
+        let response = handle_robots_txt(
+            &settings,
+            &services_with(Arc::clone(&store)),
+            &registry,
+            robots_request(),
+        )
+        .await
+        .expect("the refusal answers without asking");
+
+        assert_eq!(response.status(), StatusCode::OK);
+        assert_eq!(
+            response.headers().get(header::CACHE_CONTROL),
+            Some(&HeaderValue::from_static("public, max-age=3600"))
+        );
+        let body = response
+            .into_body()
+            .into_bytes_bounded(1024)
+            .await
+            .expect("should read the body");
+        assert_eq!(
+            body.as_ref(),
+            b"User-agent: *\nAllow: /ads.txt\nDisallow: /\n",
+            "the refusal, whatever else is selected"
+        );
+        assert!(
+            store
+                .entries
+                .lock()
+                .expect("should lock the test store")
+                .is_empty(),
+            "nothing is held for a refusal"
+        );
+    }
+
+    /// A crawler that asks with `HEAD` is told what a `GET` would be told and
+    /// is sent no file.
+    #[tokio::test]
+    async fn a_head_is_answered_with_the_headers_and_no_body() {
+        let mut settings = settings();
+        settings.robots_txt = Some(RobotsTxtConfig::refuse_all());
+        let registry = IntegrationRegistry::new(&settings).expect("should build the registry");
+        let mut request = robots_request();
+        *request.method_mut() = Method::HEAD;
+
+        let response = handle_robots_txt(
+            &settings,
+            &services_with(Arc::default()),
+            &registry,
+            request,
+        )
+        .await
+        .expect("should answer");
+
+        assert_eq!(response.status(), StatusCode::OK);
+        assert_eq!(
+            response.headers().get(header::CONTENT_TYPE),
+            Some(&HeaderValue::from_static("text/plain; charset=utf-8"))
+        );
+        let body = response
+            .into_body()
+            .into_bytes_bounded(16)
+            .await
+            .expect("should read the body");
+        assert!(body.is_empty(), "a HEAD carries no body");
+    }
+
+    /// The id of the builder that registers the echoing contributor.
+    const ECHO_BUILDER: &str = "robots-txt-echo";
+
+    /// The module name that builder is selected by.
+    const ECHO_BUILDER_MODULE: &str = "testing.robots-txt-echo";
+
+    fn register_echo(
+        _settings: &Settings,
+    ) -> Result<Option<IntegrationRegistration>, Report<TrustedServerError>> {
+        Ok(Some(
+            IntegrationRegistration::builder(ECHO_BUILDER)
+                .with_robots_txt_contributor("robots-txt.echo", Arc::new(Echoing))
+                .build(),
+        ))
+    }
+
+    fn accept(_settings: &Settings) -> Result<bool, Report<TrustedServerError>> {
+        Ok(true)
+    }
+
+    fn settings_selecting_echo() -> Settings {
+        let mut settings = settings();
+        settings.robots_txt = Some(parsed(serde_json::json!({ "modules": ["echo"] })));
+        settings
+            .insert_module_config("testing", ECHO_BUILDER_MODULE, &serde_json::json!({}))
+            .expect("should select the test builder");
+        settings
+    }
+
+    fn echo_builder() -> IntegrationBuilder {
+        IntegrationBuilder::new(ECHO_BUILDER, ECHO_BUILDER, register_echo, accept)
+            .with_module_name(ECHO_BUILDER_MODULE)
+    }
+
+    /// The handler asks a contributor for the request it is answering, being
+    /// its host and path, without the query and without the reader's
+    /// evidence.
+    #[tokio::test]
+    async fn the_handler_asks_the_contributors_for_the_request_it_answers() {
+        let settings = settings_selecting_echo();
+        let registry = IntegrationRegistry::with_registrations(&settings, &[echo_builder()])
+            .expect("should build the registry");
+
+        let response = handle_robots_txt(
+            &settings,
+            &services_with(Arc::default()),
+            &registry,
+            robots_request(),
+        )
+        .await
+        .expect("should answer");
+
+        assert_eq!(response.status(), StatusCode::OK);
+        let body = response
+            .into_body()
+            .into_bytes_bounded(4096)
+            .await
+            .expect("should read the body");
+        let file = String::from_utf8(body.to_vec()).expect("the file is text");
+        let config = settings
+            .robots_txt
+            .as_ref()
+            .expect("the settings carry a [robots-txt] section");
+        assert_eq!(file, document::assemble(config, &[rules(ECHOED)]));
+    }
+
+    /// A name nothing supplies is refused where the registry is built, so the
+    /// file does not fail on the first request for it.
+    #[test]
+    fn a_selection_nothing_supplies_is_refused_when_the_registry_is_built() {
+        let mut settings = settings();
+        settings.robots_txt = Some(parsed(serde_json::json!({ "modules": ["nobody"] })));
+
+        let error = IntegrationRegistry::new(&settings)
+            .err()
+            .expect("should refuse a contributor nothing supplies");
+        let message = format!("{error:?}");
+
+        assert!(message.contains("`nobody`"), "{message}");
+        assert!(message.contains("refuse_all"), "{message}");
+        assert!(message.contains("allow_all"), "{message}");
+    }
+}
diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs
index cae6358fab..fdee0e50c5 100644
--- a/crates/trusted-server-core/src/settings.rs
+++ b/crates/trusted-server-core/src/settings.rs
@@ -3756,6 +3756,19 @@ pub struct Settings {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     #[validate(nested)]
     pub trusted_client_ip: Option<TrustedClientIpConfig>,
+    /// Optional `robots.txt`, assembled by core from the contributors
+    /// `[robots-txt] modules` selects.
+    ///
+    /// `None` leaves `/robots.txt` to the publisher's origin, and must stay
+    /// omitted from serialized config blobs for the reason given on
+    /// `trusted_client_ip`.
+    #[serde(
+        rename = "robots-txt",
+        default,
+        skip_serializing_if = "Option::is_none"
+    )]
+    #[validate(nested)]
+    pub robots_txt: Option<crate::robots_txt::RobotsTxtConfig>,
     #[serde(default)]
     #[validate(nested)]
     pub ec: Ec,
@@ -4038,6 +4051,9 @@ impl Settings {
             .auction_html_comment_options
             .validate_metadata_keys()?;
         self.validate_asset_image_optimizer_profile_sets()?;
+        if let Some(robots_txt) = &self.robots_txt {
+            robots_txt.prepare_runtime()?;
+        }
 
         if let Some(co) = &mut self.creative_opportunities {
             co.compile_slots();
@@ -4832,6 +4848,90 @@ mod tests {
         );
     }
 
+    fn settings_str_with_robots_txt(section: &str) -> String {
+        format!("{}\n[robots-txt]\n{section}\n", crate_test_settings_str())
+    }
+
+    #[test]
+    fn the_robots_txt_section_is_read_by_its_hyphenated_name() {
+        let toml = settings_str_with_robots_txt(
+            "modules = [\"refuse_all\"]\nalways_allow = [\"/ads.txt\"]",
+        );
+
+        let settings = Settings::from_toml(&toml).expect("should read the section");
+
+        let robots_txt = settings
+            .robots_txt
+            .as_ref()
+            .expect("should carry the section");
+        assert_eq!(robots_txt.modules, vec!["refuse_all"]);
+        assert_eq!(robots_txt.always_allow, vec!["/ads.txt"]);
+        assert!(
+            settings
+                .module_sections()
+                .all(|(name, _)| name != "robots-txt"),
+            "should not also be read as the section of a module type"
+        );
+    }
+
+    #[test]
+    fn a_robots_txt_section_naming_no_module_is_refused_when_the_settings_load() {
+        let toml = settings_str_with_robots_txt("sitemap = \"https://example.com/sitemap.xml\"");
+
+        let error = Settings::from_toml(&toml).expect_err("there is no default");
+
+        assert!(
+            format!("{error:?}").contains("[robots-txt] names no module"),
+            "should say a module must be named: {error:?}"
+        );
+    }
+
+    #[test]
+    fn a_robots_txt_path_that_could_start_a_second_line_is_refused() {
+        let toml = settings_str_with_robots_txt(
+            "modules = \"refuse_all\"\nalways_allow = [\"/ads.txt\\nDisallow:\"]",
+        );
+
+        assert!(
+            Settings::from_toml(&toml).is_err(),
+            "should refuse a path that holds white space"
+        );
+    }
+
+    #[test]
+    fn settings_without_a_robots_txt_section_are_written_without_one() {
+        let settings = Settings::from_toml(&crate_test_settings_str())
+            .expect("should load the test settings fixture");
+
+        let value = serde_json::to_value(&settings).expect("should serialize the settings");
+
+        assert!(
+            value.get("robots-txt").is_none(),
+            "a build that predates the section refuses a key it does not know"
+        );
+    }
+
+    #[test]
+    fn a_robots_txt_section_survives_the_stored_form() {
+        let toml = settings_str_with_robots_txt(
+            "modules = [\"allow_all\"]\nsitemap = \"https://example.com/sitemap.xml\"",
+        );
+        let settings = Settings::from_toml(&toml).expect("should read the section");
+
+        let stored = serde_json::to_value(&settings).expect("should serialize the settings");
+        let reloaded = Settings::from_json_value(stored).expect("should read the section back");
+
+        let robots_txt = reloaded
+            .robots_txt
+            .as_ref()
+            .expect("should still carry the section");
+        assert_eq!(robots_txt.modules, vec!["allow_all"]);
+        assert_eq!(
+            robots_txt.sitemap.as_deref(),
+            Some("https://example.com/sitemap.xml")
+        );
+    }
+
     /// Mirrors the `Settings` schema of the revision that predates
     /// `trusted_client_ip`: every key that revision knew, and
     /// `deny_unknown_fields` so an extra key fails deserialization exactly as an

From 0188f5d7e9963046ef14b81b521bd23be527eadd Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 9 Oct 2026 16:13:39 +0100
Subject: [PATCH 02/15] The seam probe supplies a robots.txt contributor

Declared on the probe's one registration, beside its geo, Edge Cookie and
device modules, so an adapter's tests can select a contributor from a crate
core does not know. Its one rule names the host, path and query it was asked
for, which lets a test read back what a contributor is handed.
---
 crates/testing/seam-probe/src/lib.rs | 64 +++++++++++++++++++++++-----
 1 file changed, 54 insertions(+), 10 deletions(-)

diff --git a/crates/testing/seam-probe/src/lib.rs b/crates/testing/seam-probe/src/lib.rs
index 5f8365bd6f..c836a78c3d 100644
--- a/crates/testing/seam-probe/src/lib.rs
+++ b/crates/testing/seam-probe/src/lib.rs
@@ -2,11 +2,12 @@
 //! part of the integration seam from a vendor crate's position.
 //!
 //! One registration carries a browser module, a proxy route, a geo module,
-//! an Edge Cookie identity module, a device module and a middleware for
-//! each phase of a page, alongside its own configuration block, and the builder
-//! adds a request preparer, a demand implementation `[demand]` can name and
-//! an ad server implementation `[ad-server]` can name. The round-trip
-//! tests in `crates/trusted-server-adapter-axum/tests/seam_probe.rs` and
+//! an Edge Cookie identity module, a device module, a robots.txt contributor
+//! and a middleware for each phase of a page, alongside its own configuration
+//! block, and the builder adds a request preparer, a demand implementation
+//! `[demand]` can name and an ad server implementation `[ad-server]` can
+//! name. The round-trip tests in
+//! `crates/trusted-server-adapter-axum/tests/seam_probe.rs` and
 //! `crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs` drive
 //! each of those through a real adapter, so the seam is proven by a caller that
 //! core does not know about.
@@ -47,10 +48,11 @@ use trusted_server_core::integrations::{
 use trusted_server_core::middleware::{
     AttributeRewrite, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, TextHandler,
 };
-use trusted_server_core::module_context::ModuleCall;
+use trusted_server_core::module_context::{ModuleCall, ModuleRequest};
 use trusted_server_core::platform::{
     GeoInfo, PlatformError, PlatformGeo, PlatformResponse, RuntimeServices,
 };
+use trusted_server_core::robots_txt::{Contribution, RobotsTxtContributor};
 use trusted_server_core::settings::{IntegrationConfig, Settings};
 use validator::Validate;
 
@@ -60,12 +62,16 @@ pub const SEAM_PROBE_ID: &str = "seam_probe";
 /// The name this module is selected by, its crate folder below `crates/`, so
 /// `[testing] modules = ["seam-probe"]` selects it. Its geo, Edge Cookie and
 /// device modules carry the same name, so `[geo] module`, `[ec] module` and
-/// `[device] module` select each of them as `testing.seam-probe`.
+/// `[device] module` select each of them as `testing.seam-probe`, and so does
+/// `[robots-txt] modules` its robots.txt contributor.
 #[must_use]
 pub fn module_name() -> &'static str {
     trusted_server_core::module_name!()
 }
 
+/// The crawler the probe's robots.txt rules are addressed to.
+pub const SEAM_PROBE_ROBOTS_AGENT: &str = "SeamProbeBot";
+
 /// Source label the registry uses in duplicate-id errors.
 pub const SEAM_PROBE_SOURCE: &str = "trusted-server-integration-seam-probe";
 
@@ -372,6 +378,42 @@ impl DeviceModule for SeamProbeDevice {
     }
 }
 
+/// The probe's robots.txt contributor, declared on the same registration.
+///
+/// Its one rule names the request it was asked for, so a test can read back
+/// that the contributor was handed the host and the path and not the query.
+#[derive(Debug)]
+pub struct SeamProbeRobotsTxt;
+
+impl SeamProbeRobotsTxt {
+    fn rules(&self, request: ModuleRequest<'_>) -> Contribution {
+        Contribution::parse(&format!(
+            "User-Agent: {SEAM_PROBE_ROBOTS_AGENT}\nDisallow: /{}{}?{}",
+            request.host(),
+            request.path(),
+            request.query()
+        ))
+    }
+}
+
+#[async_trait::async_trait(?Send)]
+impl RobotsTxtContributor for SeamProbeRobotsTxt {
+    fn refresh(&self) -> core::time::Duration {
+        core::time::Duration::from_secs(60)
+    }
+
+    fn fingerprint(&self) -> String {
+        "seam-probe".to_owned()
+    }
+
+    async fn contribute(
+        &self,
+        call: ModuleCall<'_>,
+    ) -> Result<Contribution, Report<TrustedServerError>> {
+        Ok(call.inject(self, Self::rules)?)
+    }
+}
+
 #[async_trait::async_trait(?Send)]
 impl PlatformGeo for SeamProbeGeo {
     async fn lookup(
@@ -639,11 +681,13 @@ pub fn register(
             .with_geo_module(module_name(), Arc::new(SeamProbeGeo::new(config.country)));
     }
     {
-        // Identity and device are declared the same way location is, so one
-        // module supplies all three and there is no second mechanism.
+        // Identity, device and robots.txt rules are declared the same way
+        // location is, so one module supplies all four and there is no second
+        // mechanism.
         registration = registration
             .with_ec_module(module_name(), Arc::new(SeamProbeEc))
-            .with_device_module(module_name(), Arc::new(SeamProbeDevice));
+            .with_device_module(module_name(), Arc::new(SeamProbeDevice))
+            .with_robots_txt_contributor(module_name(), Arc::new(SeamProbeRobotsTxt));
     }
 
     Ok(Some(registration.build()))

From 3701fbc4c1b36575e04eedf76da518a8bbca0e06 Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 9 Oct 2026 16:13:40 +0100
Subject: [PATCH 03/15] Every adapter serves /robots.txt and the refusal's
 response header

The route is registered only when the settings carry a `[robots-txt]`
section, for `GET` and `HEAD`, beside each adapter's other fixed routes. On
Fastly and Axum it is a row of the named-route table, gated by
`named_route_is_served`, so a request for it passes what every named
route's request passes there. Every other method on the path goes to the
publisher fallback. Without the section the path reaches the publisher's
origin like any other, so a publisher keeping their own file is left alone.

With `refuse_all` selected every response an adapter finalizes carries
`X-Robots-Tag: noindex, nofollow`. It is written last when a response is
finalized, and replaces any tag already there, so that neither an origin's
header nor an operator's `[response_headers]` can loosen it.

The comment on the Axum adapter's one preparation point now names the
attestation pages as the other route outside it.
---
 crates/trusted-server-adapter-axum/src/app.rs |  36 +++-
 .../src/middleware.rs                         |   4 +
 .../tests/seam_probe.rs                       | 155 +++++++++++++++++
 .../src/app.rs                                |  18 ++
 .../src/middleware.rs                         |   4 +
 .../tests/routes.rs                           | 159 ++++++++++++++++++
 .../trusted-server-adapter-fastly/src/app.rs  |  27 +++
 .../src/app/seam_probe_tests.rs               | 139 +++++++++++++++
 .../src/middleware.rs                         |   6 +
 crates/trusted-server-adapter-spin/src/app.rs |  27 +++
 .../src/middleware.rs                         |   4 +
 .../tests/routes.rs                           | 159 ++++++++++++++++++
 12 files changed, 734 insertions(+), 4 deletions(-)

diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs
index daca62627e..38e65c0c40 100644
--- a/crates/trusted-server-adapter-axum/src/app.rs
+++ b/crates/trusted-server-adapter-axum/src/app.rs
@@ -39,6 +39,7 @@ use trusted_server_core::publisher::{
 use trusted_server_core::request_signing::{
     handle_trusted_server_discovery, handle_verify_signature,
 };
+use trusted_server_core::robots_txt::{ROBOTS_TXT_PATH, handle_robots_txt};
 use trusted_server_core::settings::Settings;
 use trusted_server_core::settings_data::{
     default_config_key, default_config_store_name, get_settings_from_config_store_with,
@@ -282,9 +283,10 @@ where
     let services = state.services_for_request(&ctx);
     let mut req = ctx.into_request();
     // The single preparation point for the adapter. Every route in the table
-    // except `/health` is registered to `named_route_handler` or
-    // `fallback_handler`, and both wrap this function, so each request has its
-    // modules' preparers run exactly once and always before routing.
+    // except `/health` and the attestation pages is registered to
+    // `named_route_handler` or `fallback_handler`, and both wrap this
+    // function, so each request has its modules' preparers run exactly once
+    // and always before routing.
     if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) {
         return Ok(http_error(&error));
     }
@@ -442,6 +444,7 @@ enum NamedRouteHandler {
     FirstPartyClick,
     FirstPartySign,
     FirstPartyProxyRebuild,
+    RobotsTxt,
 }
 
 struct NamedRoute {
@@ -450,7 +453,7 @@ struct NamedRoute {
     handler: NamedRouteHandler,
 }
 
-fn named_routes() -> [NamedRoute; 14] {
+fn named_routes() -> [NamedRoute; 15] {
     [
         NamedRoute {
             path: "/.well-known/trusted-server.json",
@@ -535,9 +538,28 @@ fn named_routes() -> [NamedRoute; 14] {
             primary_methods: &[Method::GET, Method::POST],
             handler: NamedRouteHandler::FirstPartyProxyRebuild,
         },
+        // The publisher's robots.txt, served only when the settings carry a
+        // `[robots-txt]` section, see `named_route_is_served`.
+        NamedRoute {
+            path: ROBOTS_TXT_PATH,
+            primary_methods: &[Method::GET, Method::HEAD],
+            handler: NamedRouteHandler::RobotsTxt,
+        },
     ]
 }
 
+/// Whether a named route is served for these settings.
+///
+/// `/robots.txt` is this server's only when the publisher's settings carry a
+/// `[robots-txt]` section. Without one the path reaches the publisher's
+/// origin like any other, so a publisher keeping their own file is left alone.
+fn named_route_is_served(settings: &Settings, handler: NamedRouteHandler) -> bool {
+    match handler {
+        NamedRouteHandler::RobotsTxt => settings.robots_txt.is_some(),
+        _ => true,
+    }
+}
+
 fn named_route_handler(
     state: Arc<AppState>,
     handler: NamedRouteHandler,
@@ -612,6 +634,9 @@ fn named_route_handler(
                     NamedRouteHandler::FirstPartyProxyRebuild => {
                         handle_first_party_proxy_rebuild(&state.settings, &services, req).await
                     }
+                    NamedRouteHandler::RobotsTxt => {
+                        handle_robots_txt(&state.settings, &services, &state.registry, req).await
+                    }
                 }
             },
         ))
@@ -791,6 +816,9 @@ fn build_router(state: &Arc<AppState>) -> RouterService {
     });
 
     for route in named_routes() {
+        if !named_route_is_served(&state.settings, route.handler) {
+            continue;
+        }
         for method in route.primary_methods {
             router = router.route(
                 route.path,
diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs
index 9690a7d04d..cbdf1663e6 100644
--- a/crates/trusted-server-adapter-axum/src/middleware.rs
+++ b/crates/trusted-server-adapter-axum/src/middleware.rs
@@ -99,6 +99,10 @@ pub(crate) fn apply_finalize_headers(settings: &Settings, response: &mut Respons
     trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy(
         settings, response,
     );
+
+    // Last, so that nothing an origin or an operator's header set can loosen
+    // a site taken out of search.
+    trusted_server_core::robots_txt::apply_response_robots_tag(settings, response);
 }
 
 // ---------------------------------------------------------------------------
diff --git a/crates/trusted-server-adapter-axum/tests/seam_probe.rs b/crates/trusted-server-adapter-axum/tests/seam_probe.rs
index a19a9b23b0..bb815ed33f 100644
--- a/crates/trusted-server-adapter-axum/tests/seam_probe.rs
+++ b/crates/trusted-server-adapter-axum/tests/seam_probe.rs
@@ -256,6 +256,161 @@ async fn proxy_route_reports_the_modules_geo_and_that_the_preparer_ran() {
     );
 }
 
+/// `[robots-txt] modules` naming the probe serves the rules the probe gives,
+/// asked for the request's host and path and never its query, because the
+/// answer is held and served to every crawler.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn robots_txt_selection_naming_a_module_serves_the_rules_it_gives() {
+    let settings = settings_with(&format!(
+        r#"
+            [robots-txt]
+            modules = ["testing.seam-probe", "allow_all"]
+            sitemap = "https://test-publisher.example.com/sitemap.xml"
+            {PROBE_BLOCK}
+        "#
+    ));
+    let mut service = service_with(settings, &[seam_probe::builder()]);
+
+    let response = get(&mut service, "/robots.txt?crawler=one").await;
+
+    assert_eq!(
+        response.status().as_u16(),
+        200,
+        "the file should be served here and not by the origin"
+    );
+    assert_eq!(
+        response
+            .headers()
+            .get("content-type")
+            .and_then(|value| value.to_str().ok()),
+        Some("text/plain; charset=utf-8")
+    );
+    let file = body_text(response).await;
+    let agent = seam_probe::SEAM_PROBE_ROBOTS_AGENT;
+    assert!(
+        file.starts_with(&format!("User-Agent: {agent}\nDisallow: /")),
+        "the probe's rules should come first, as `modules` orders them: {file}"
+    );
+    assert!(
+        file.contains("/robots.txt?\n"),
+        "the probe should be asked for the path with no query: {file}"
+    );
+    assert!(
+        !file.contains("crawler=one"),
+        "nothing one crawler put in the address should reach a held answer: {file}"
+    );
+    assert!(
+        file.ends_with(
+            "User-Agent: *\nAllow: /\n\nSitemap: https://test-publisher.example.com/sitemap.xml\n"
+        ),
+        "core's own rules and the sitemap should follow: {file}"
+    );
+}
+
+/// With `refuse_all` selected the file is the refusal, and every response
+/// this adapter finalizes says not to index, the response of another route
+/// included.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn refusing_every_crawler_tags_every_response() {
+    let settings = settings_with(&format!(
+        r#"
+            [robots-txt]
+            modules = "refuse_all"
+            always_allow = ["/ads.txt"]
+            {PROBE_BLOCK}
+        "#
+    ));
+    let mut service = service_with(settings, &[seam_probe::builder()]);
+    let tag = |response: &axum::response::Response| {
+        response
+            .headers()
+            .get("x-robots-tag")
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned)
+    };
+
+    let file = get(&mut service, "/robots.txt").await;
+
+    assert_eq!(file.status().as_u16(), 200, "the refusal should be served");
+    assert_eq!(tag(&file).as_deref(), Some("noindex, nofollow"));
+    assert_eq!(
+        body_text(file).await,
+        "User-agent: *\nAllow: /ads.txt\nDisallow: /\n"
+    );
+
+    let other = get(&mut service, seam_probe::SEAM_PROBE_REPORT_PATH).await;
+
+    assert_eq!(
+        tag(&other).as_deref(),
+        Some("noindex, nofollow"),
+        "no route may be the one page a crawler is allowed to index"
+    );
+}
+
+/// A name `[robots-txt] modules` gives that nothing the deployment runs
+/// supplies stops the adapter at startup, so the file does not fail on the
+/// first crawler to ask for it.
+#[test]
+fn robots_txt_selection_naming_a_module_that_is_not_running_fails_at_startup() {
+    let settings = settings_with(
+        r#"
+            [robots-txt]
+            modules = ["testing.seam-probe"]
+        "#,
+    );
+
+    let error = TrustedServerApp::routes_with_registrations(settings, &[seam_probe::builder()])
+        .err()
+        .expect("should refuse a contributor whose module no section selects");
+    let message = format!("{error:?}");
+
+    assert!(
+        message.contains("`[robots-txt] modules` names `testing.seam-probe`"),
+        "should name the selection: {message}"
+    );
+    assert!(
+        message.contains("refuse_all") && message.contains("allow_all"),
+        "should say what can be named: {message}"
+    );
+}
+
+/// The methods this adapter registers for `/robots.txt` under `settings`.
+fn robots_txt_methods(settings: Settings) -> Vec<String> {
+    TrustedServerApp::routes_with_registrations(settings, &[seam_probe::builder()])
+        .expect("should build the routes")
+        .routes()
+        .into_iter()
+        .filter(|route| route.path() == "/robots.txt")
+        .map(|route| route.method().to_string())
+        .collect()
+}
+
+/// With no `[robots-txt]` section the path is the publisher's, and the probe
+/// being on offer changes nothing. With the section the path is this
+/// server's for `GET` and `HEAD`.
+#[test]
+fn robots_txt_is_this_servers_only_when_the_settings_carry_the_section() {
+    assert_eq!(
+        robots_txt_methods(settings_with(PROBE_BLOCK)),
+        Vec::<String>::new(),
+        "without the section no route of this server names the path"
+    );
+
+    let methods = robots_txt_methods(settings_with(&format!(
+        r#"
+            [robots-txt]
+            modules = "refuse_all"
+            {PROBE_BLOCK}
+        "#
+    )));
+    for method in ["GET", "HEAD"] {
+        assert!(
+            methods.iter().any(|registered| registered == method),
+            "{method} /robots.txt should be answered here: {methods:?}"
+        );
+    }
+}
+
 /// A request prepares exactly once whether it is served by a named route or by
 /// the fallback, so the invariant holds across the whole route table rather
 /// than only on the path the probe's own proxy route sits on.
diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs
index 6f5cf17981..31a5356ec4 100644
--- a/crates/trusted-server-adapter-cloudflare/src/app.rs
+++ b/crates/trusted-server-adapter-cloudflare/src/app.rs
@@ -45,6 +45,7 @@ use trusted_server_core::publisher::{
 use trusted_server_core::request_signing::{
     handle_trusted_server_discovery, handle_verify_signature,
 };
+use trusted_server_core::robots_txt::{ROBOTS_TXT_PATH, handle_robots_txt};
 use trusted_server_core::settings::Settings;
 
 use crate::middleware::{FinalizeResponseMiddleware, SanitizeRequestMiddleware};
@@ -891,6 +892,23 @@ fn build_router(state: &Arc<AppState>) -> RouterService {
             }
         }
 
+        // `/robots.txt` is this server's only when the settings carry a
+        // `[robots-txt]` section. Without one the path reaches the publisher's
+        // origin like any other, so a publisher keeping their own file is
+        // left alone.
+        if state.settings.robots_txt.is_some() {
+            let robots_txt = make_handler(Arc::clone(&state), |s, services, req| async move {
+                handle_robots_txt(&s.settings, &services, &s.registry, req).await
+            });
+            for method in publisher_fallback_methods() {
+                if matches!(method, Method::GET | Method::HEAD) {
+                    router = router.route(ROBOTS_TXT_PATH, method, robots_txt.clone());
+                } else {
+                    router = router.route(ROBOTS_TXT_PATH, method, fallback.clone());
+                }
+            }
+        }
+
         for method in publisher_fallback_methods() {
             router = router.route("/", method.clone(), fallback.clone());
             router = router.route("/{*rest}", method, fallback.clone());
diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs
index 64c086a5ef..8db66921bc 100644
--- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs
+++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs
@@ -112,6 +112,10 @@ pub(crate) fn apply_finalize_headers(
     trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy(
         settings, response,
     );
+
+    // Last, so that nothing an origin or an operator's header set can loosen
+    // a site taken out of search.
+    trusted_server_core::robots_txt::apply_response_robots_tag(settings, response);
 }
 
 // ---------------------------------------------------------------------------
diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs
index 8f803609dd..fae32e6683 100644
--- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs
+++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs
@@ -569,3 +569,162 @@ fn every_fixed_route_beneath_an_underscore_prefix_is_reserved() {
         }
     }
 }
+
+// ---------------------------------------------------------------------------
+// robots.txt
+// ---------------------------------------------------------------------------
+
+/// The route test settings with a `[robots-txt]` section that refuses every
+/// crawler and keeps one path open.
+fn router_refusing_every_crawler() -> RouterService {
+    let settings = Settings::from_toml(
+        r#"
+            [publisher]
+            domain = "test-publisher.example.com"
+            cookie_domain = ".test-publisher.example.com"
+            origin_url = "https://origin.test-publisher.example.com"
+            proxy_secret = "route-test-proxy-secret"
+
+            [ec]
+            module = "hmac"
+
+            [ec.hmac]
+            passphrase = "test-secret-key-32-bytes-minimum"
+
+            [geo]
+            assume_single_jurisdiction = true
+
+            [robots-txt]
+            modules = "refuse_all"
+            always_allow = ["/ads.txt"]
+        "#,
+    )
+    .expect("should parse route test settings");
+
+    TrustedServerApp::routes_with_settings(settings)
+        .expect("should build router from test settings")
+}
+
+fn robots_request(method: &str, path: &str) -> Request {
+    request_builder()
+        .method(method)
+        .uri(path)
+        .body(edgezero_core::body::Body::empty())
+        .expect("should build request")
+}
+
+fn robots_tag(response: &Response) -> Option<String> {
+    response
+        .headers()
+        .get("x-robots-tag")
+        .and_then(|value| value.to_str().ok())
+        .map(str::to_owned)
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn refusing_every_crawler_serves_the_refusal_and_tags_every_response() {
+    let file = route(
+        router_refusing_every_crawler(),
+        robots_request("GET", "/robots.txt"),
+    )
+    .await;
+
+    assert_eq!(
+        file.status().as_u16(),
+        200,
+        "the refusal should be served here"
+    );
+    assert_eq!(robots_tag(&file).as_deref(), Some("noindex, nofollow"));
+    let body = file.into_body().into_bytes().unwrap_or_default();
+    assert_eq!(&body[..], b"User-agent: *\nAllow: /ads.txt\nDisallow: /\n");
+
+    let other = route(
+        router_refusing_every_crawler(),
+        robots_request("GET", "/.well-known/trusted-server.json"),
+    )
+    .await;
+
+    assert_eq!(
+        robots_tag(&other).as_deref(),
+        Some("noindex, nofollow"),
+        "no route may be the one page a crawler is allowed to index"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_head_of_robots_txt_is_answered_here_with_no_body() {
+    let response = route(
+        router_refusing_every_crawler(),
+        robots_request("HEAD", "/robots.txt"),
+    )
+    .await;
+
+    assert_eq!(
+        response.status().as_u16(),
+        200,
+        "a HEAD should be answered here as a GET is"
+    );
+    assert_eq!(
+        response
+            .headers()
+            .get("content-type")
+            .and_then(|value| value.to_str().ok()),
+        Some("text/plain; charset=utf-8")
+    );
+    let body = response.into_body().into_bytes().unwrap_or_default();
+    assert!(body.is_empty(), "a HEAD carries no body");
+}
+
+/// A method that is neither `GET` nor `HEAD` is the publisher's, as it is on
+/// any other path, so it is not answered with the file.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn another_method_on_robots_txt_is_not_answered_with_the_file() {
+    let response = route(
+        router_refusing_every_crawler(),
+        robots_request("POST", "/robots.txt"),
+    )
+    .await;
+
+    let body = response.into_body().into_bytes().unwrap_or_default();
+    assert!(
+        !body.starts_with(b"User-agent:"),
+        "a POST should reach the publisher fallback and not be given the file"
+    );
+}
+
+#[test]
+fn robots_txt_is_this_servers_only_when_the_settings_carry_the_section() {
+    let registered = |router: RouterService| -> Vec<(String, String)> {
+        router
+            .routes()
+            .into_iter()
+            .map(|route| (route.method().to_string(), route.path().to_string()))
+            .collect()
+    };
+
+    let without = registered(test_router());
+    assert!(
+        !without.iter().any(|(_, path)| path == "/robots.txt"),
+        "without the section the path is the publisher's: {without:?}"
+    );
+
+    let with = registered(router_refusing_every_crawler());
+    for method in ["GET", "HEAD"] {
+        assert!(
+            with.iter()
+                .any(|(registered, path)| registered == method && path == "/robots.txt"),
+            "{method} /robots.txt should be answered here: {with:?}"
+        );
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_response_carries_no_blanket_rule_without_the_section() {
+    let response = route(
+        test_router(),
+        robots_request("GET", "/.well-known/trusted-server.json"),
+    )
+    .await;
+
+    assert_eq!(robots_tag(&response), None);
+}
diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs
index cc412dce00..9f13334f4c 100644
--- a/crates/trusted-server-adapter-fastly/src/app.rs
+++ b/crates/trusted-server-adapter-fastly/src/app.rs
@@ -141,6 +141,7 @@ use trusted_server_core::publisher::{
 use trusted_server_core::request_signing::{
     handle_trusted_server_discovery, handle_verify_signature,
 };
+use trusted_server_core::robots_txt::{ROBOTS_TXT_PATH, handle_robots_txt};
 use trusted_server_core::settings::{ProxyAssetRoute, Settings};
 use trusted_server_core::settings_data::{
     DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store_with,
@@ -948,6 +949,9 @@ async fn run_named_route(
         NamedRouteHandler::FirstPartyProxyRebuild => {
             handle_first_party_proxy_rebuild(&state.settings, services, req).await
         }
+        NamedRouteHandler::RobotsTxt => {
+            handle_robots_txt(&state.settings, services, &state.registry, req).await
+        }
     }
 }
 
@@ -1301,6 +1305,7 @@ enum NamedRouteHandler {
     FirstPartyClick,
     FirstPartySign,
     FirstPartyProxyRebuild,
+    RobotsTxt,
 }
 
 struct NamedRoute {
@@ -1419,8 +1424,27 @@ const NAMED_ROUTES: &[NamedRoute] = &[
         primary_methods: &[Method::GET, Method::POST],
         handler: NamedRouteHandler::FirstPartyProxyRebuild,
     },
+    // The publisher's robots.txt, served only when the settings carry a
+    // `[robots-txt]` section, see `named_route_is_served`.
+    NamedRoute {
+        path: ROBOTS_TXT_PATH,
+        primary_methods: &[Method::GET, Method::HEAD],
+        handler: NamedRouteHandler::RobotsTxt,
+    },
 ];
 
+/// Whether a named route is served for these settings.
+///
+/// `/robots.txt` is this server's only when the publisher's settings carry a
+/// `[robots-txt]` section. Without one the path reaches the publisher's
+/// origin like any other, so a publisher keeping their own file is left alone.
+fn named_route_is_served(settings: &Settings, handler: NamedRouteHandler) -> bool {
+    match handler {
+        NamedRouteHandler::RobotsTxt => settings.robots_txt.is_some(),
+        _ => true,
+    }
+}
+
 fn named_route_handler(
     state: Arc<AppState>,
     handler: NamedRouteHandler,
@@ -1519,6 +1543,9 @@ impl TrustedServerApp {
         // non-primary publisher fallback method is registered from the same
         // row. Adding a named route now requires editing only this table.
         for route in NAMED_ROUTES {
+            if !named_route_is_served(&state.settings, route.handler) {
+                continue;
+            }
             for method in route.primary_methods {
                 router = router.route(
                     route.path,
diff --git a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs
index 4ac37fcdab..39ad2d99f7 100644
--- a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs
+++ b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs
@@ -208,6 +208,145 @@ fn proxy_route_reports_the_modules_geo_and_that_the_preparer_ran() {
     );
 }
 
+/// `[robots-txt] modules` naming the probe serves the rules the probe gives,
+/// asked for the request's host and path and never its query, because the
+/// answer is held and served to every crawler.
+#[test]
+fn robots_txt_selection_naming_a_module_serves_the_rules_it_gives() {
+    let settings = settings_with(&format!(
+        r#"
+        [robots-txt]
+        modules = ["testing.seam-probe", "allow_all"]
+        {HMAC_BLOCK}
+        {PROBE_BLOCK}
+        "#
+    ));
+    let router = router_with(settings, &[seam_probe::builder()]);
+
+    let response = get(&router, "/robots.txt?crawler=one", None);
+
+    assert_eq!(
+        response.status().as_u16(),
+        200,
+        "the file should be served here and not by the origin"
+    );
+    assert!(
+        response.headers().get("x-robots-tag").is_none(),
+        "a file made from contributors puts no blanket rule on responses"
+    );
+    let file = body_text(response);
+    let agent = seam_probe::SEAM_PROBE_ROBOTS_AGENT;
+    assert!(
+        file.starts_with(&format!("User-Agent: {agent}\nDisallow: /")),
+        "the probe's rules should come first, as `modules` orders them: {file}"
+    );
+    assert!(
+        file.contains("/robots.txt?\n"),
+        "the probe should be asked for the path with no query: {file}"
+    );
+    assert!(
+        !file.contains("crawler=one"),
+        "nothing one crawler put in the address should reach a held answer: {file}"
+    );
+    assert!(
+        file.ends_with("User-Agent: *\nAllow: /\n"),
+        "core's own rules should follow: {file}"
+    );
+}
+
+/// With `refuse_all` selected the file is the refusal, and every response
+/// this adapter finalizes says not to index, the response of another route
+/// included.
+#[test]
+fn refusing_every_crawler_tags_every_response() {
+    let settings = settings_with(&format!(
+        r#"
+        [robots-txt]
+        modules = "refuse_all"
+        always_allow = ["/ads.txt"]
+        {HMAC_BLOCK}
+        {PROBE_BLOCK}
+        "#
+    ));
+    let router = router_with(settings, &[seam_probe::builder()]);
+    let tag = |response: &Response| {
+        response
+            .headers()
+            .get("x-robots-tag")
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned)
+    };
+
+    let file = get(&router, "/robots.txt", None);
+
+    assert_eq!(file.status().as_u16(), 200, "the refusal should be served");
+    assert_eq!(tag(&file).as_deref(), Some("noindex, nofollow"));
+    assert_eq!(
+        body_text(file),
+        "User-agent: *\nAllow: /ads.txt\nDisallow: /\n"
+    );
+
+    let other = get(&router, seam_probe::SEAM_PROBE_REPORT_PATH, None);
+
+    assert_eq!(
+        tag(&other).as_deref(),
+        Some("noindex, nofollow"),
+        "no route may be the one page a crawler is allowed to index"
+    );
+}
+
+/// The methods this adapter registers for `/robots.txt`.
+fn robots_txt_methods(router: &RouterService) -> Vec<String> {
+    router
+        .routes()
+        .into_iter()
+        .filter(|route| route.path() == "/robots.txt")
+        .map(|route| route.method().to_string())
+        .collect()
+}
+
+/// With no `[robots-txt]` section the path is the publisher's, and the probe
+/// being on offer changes nothing. With the section the path is this
+/// server's for `GET` and `HEAD`.
+#[test]
+fn robots_txt_is_this_servers_only_when_the_settings_carry_the_section() {
+    let without = router_with(
+        settings_with(&format!("{HMAC_BLOCK}{PROBE_BLOCK}")),
+        &[seam_probe::builder()],
+    );
+
+    assert_eq!(
+        robots_txt_methods(&without),
+        Vec::<String>::new(),
+        "without the section no route of this server names the path"
+    );
+    let other = get(&without, seam_probe::SEAM_PROBE_REPORT_PATH, None);
+    assert!(
+        other.headers().get("x-robots-tag").is_none(),
+        "no blanket rule is placed on responses without the section"
+    );
+
+    let with = router_with(
+        settings_with(&format!(
+            r#"
+            [robots-txt]
+            modules = "refuse_all"
+            {HMAC_BLOCK}
+            {PROBE_BLOCK}
+            "#
+        )),
+        &[seam_probe::builder()],
+    );
+
+    let methods = robots_txt_methods(&with);
+    for method in ["GET", "HEAD"] {
+        assert!(
+            methods.iter().any(|registered| registered == method),
+            "{method} /robots.txt should be answered here: {methods:?}"
+        );
+    }
+}
+
 /// A request prepares exactly once whether a named route or the fallback
 /// serves it.
 #[test]
diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs
index 3c1f8bd663..9d3f988c20 100644
--- a/crates/trusted-server-adapter-fastly/src/middleware.rs
+++ b/crates/trusted-server-adapter-fastly/src/middleware.rs
@@ -156,6 +156,8 @@ pub(crate) fn geo_allowed_for_response(response: &Response) -> bool {
 /// 5. `settings.response_headers` — operator-configured overrides, except the
 ///    cache-controlling headers are skipped on uncacheable (`private`/`no-store`)
 ///    responses so operators cannot re-enable shared caching for per-user payloads
+/// 6. `X-Robots-Tag: noindex, nofollow` when `[robots-txt] modules` selects
+///    `refuse_all`
 pub(crate) fn apply_finalize_headers(
     settings: &Settings,
     geo_info: Option<&GeoInfo>,
@@ -192,6 +194,10 @@ pub(crate) fn apply_finalize_headers(
     trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy(
         settings, response,
     );
+
+    // Last, so that nothing an origin or an operator's header set can loosen
+    // a site taken out of search.
+    trusted_server_core::robots_txt::apply_response_robots_tag(settings, response);
 }
 
 /// Forces cookie-bearing responses to stay private to shared caches.
diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs
index e69f37a1eb..8a12be44bb 100644
--- a/crates/trusted-server-adapter-spin/src/app.rs
+++ b/crates/trusted-server-adapter-spin/src/app.rs
@@ -48,6 +48,7 @@ use trusted_server_core::publisher::{
 use trusted_server_core::request_signing::{
     handle_trusted_server_discovery, handle_verify_signature,
 };
+use trusted_server_core::robots_txt::{ROBOTS_TXT_PATH, handle_robots_txt};
 use trusted_server_core::settings::Settings;
 #[cfg(all(feature = "spin", target_arch = "wasm32"))]
 use trusted_server_core::settings_data::{default_config_key, default_secret_store_name};
@@ -1106,6 +1107,32 @@ fn build_router(state: &Arc<AppState>) -> RouterService {
             }
         }
 
+        // `/robots.txt` is this server's only when the settings carry a
+        // `[robots-txt]` section. Without one the path reaches the publisher's
+        // origin like any other, so a publisher keeping their own file is
+        // left alone.
+        if state.settings.robots_txt.is_some() {
+            let s = Arc::clone(&state);
+            let robots_txt_handler = move |ctx: RequestContext| {
+                let s = Arc::clone(&s);
+                async move {
+                    let services = s.services_for_request(&ctx);
+                    Ok::<Response, EdgeError>(
+                        handle_robots_txt(&s.settings, &services, &s.registry, ctx.into_request())
+                            .await
+                            .unwrap_or_else(|e| http_error(&e)),
+                    )
+                }
+            };
+            for method in publisher_fallback_methods() {
+                if matches!(method, Method::GET | Method::HEAD) {
+                    builder = builder.route(ROBOTS_TXT_PATH, method, robots_txt_handler.clone());
+                } else {
+                    builder = builder.route(ROBOTS_TXT_PATH, method, fallback.clone());
+                }
+            }
+        }
+
         for method in publisher_fallback_methods() {
             builder = builder.route("/", method.clone(), fallback.clone());
             builder = builder.route("/{*rest}", method, fallback.clone());
diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs
index 9b73c4f227..db5f90d60e 100644
--- a/crates/trusted-server-adapter-spin/src/middleware.rs
+++ b/crates/trusted-server-adapter-spin/src/middleware.rs
@@ -136,6 +136,10 @@ pub(crate) fn apply_finalize_headers(
     trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy(
         settings, response,
     );
+
+    // Last, so that nothing an origin or an operator's header set can loosen
+    // a site taken out of search.
+    trusted_server_core::robots_txt::apply_response_robots_tag(settings, response);
 }
 
 // ---------------------------------------------------------------------------
diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs
index b6c5378c05..28127614cd 100644
--- a/crates/trusted-server-adapter-spin/tests/routes.rs
+++ b/crates/trusted-server-adapter-spin/tests/routes.rs
@@ -907,3 +907,162 @@ fn every_fixed_route_beneath_an_underscore_prefix_is_reserved() {
         }
     }
 }
+
+// ---------------------------------------------------------------------------
+// robots.txt
+// ---------------------------------------------------------------------------
+
+/// The route test settings with a `[robots-txt]` section that refuses every
+/// crawler and keeps one path open.
+fn router_refusing_every_crawler() -> RouterService {
+    let settings = Settings::from_toml(
+        r#"
+            [publisher]
+            domain = "test-publisher.example.com"
+            cookie_domain = ".test-publisher.example.com"
+            origin_url = "https://origin.test-publisher.example.com"
+            proxy_secret = "route-test-proxy-secret"
+
+            [ec]
+            module = "hmac"
+
+            [ec.hmac]
+            passphrase = "test-secret-key-32-bytes-minimum"
+
+            [geo]
+            assume_single_jurisdiction = true
+
+            [robots-txt]
+            modules = "refuse_all"
+            always_allow = ["/ads.txt"]
+        "#,
+    )
+    .expect("should parse route test settings");
+
+    TrustedServerApp::routes_with_settings(settings)
+        .expect("should build router from test settings")
+}
+
+fn robots_request(method: &str, path: &str) -> Request {
+    request_builder()
+        .method(method)
+        .uri(path)
+        .body(edgezero_core::body::Body::empty())
+        .expect("should build request")
+}
+
+fn robots_tag(response: &Response) -> Option<String> {
+    response
+        .headers()
+        .get("x-robots-tag")
+        .and_then(|value| value.to_str().ok())
+        .map(str::to_owned)
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn refusing_every_crawler_serves_the_refusal_and_tags_every_response() {
+    let file = route(
+        router_refusing_every_crawler(),
+        robots_request("GET", "/robots.txt"),
+    )
+    .await;
+
+    assert_eq!(
+        file.status().as_u16(),
+        200,
+        "the refusal should be served here"
+    );
+    assert_eq!(robots_tag(&file).as_deref(), Some("noindex, nofollow"));
+    let body = file.into_body().into_bytes().unwrap_or_default();
+    assert_eq!(&body[..], b"User-agent: *\nAllow: /ads.txt\nDisallow: /\n");
+
+    let other = route(
+        router_refusing_every_crawler(),
+        robots_request("GET", "/.well-known/trusted-server.json"),
+    )
+    .await;
+
+    assert_eq!(
+        robots_tag(&other).as_deref(),
+        Some("noindex, nofollow"),
+        "no route may be the one page a crawler is allowed to index"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_head_of_robots_txt_is_answered_here_with_no_body() {
+    let response = route(
+        router_refusing_every_crawler(),
+        robots_request("HEAD", "/robots.txt"),
+    )
+    .await;
+
+    assert_eq!(
+        response.status().as_u16(),
+        200,
+        "a HEAD should be answered here as a GET is"
+    );
+    assert_eq!(
+        response
+            .headers()
+            .get("content-type")
+            .and_then(|value| value.to_str().ok()),
+        Some("text/plain; charset=utf-8")
+    );
+    let body = response.into_body().into_bytes().unwrap_or_default();
+    assert!(body.is_empty(), "a HEAD carries no body");
+}
+
+/// A method that is neither `GET` nor `HEAD` is the publisher's, as it is on
+/// any other path, so it is not answered with the file.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn another_method_on_robots_txt_is_not_answered_with_the_file() {
+    let response = route(
+        router_refusing_every_crawler(),
+        robots_request("POST", "/robots.txt"),
+    )
+    .await;
+
+    let body = response.into_body().into_bytes().unwrap_or_default();
+    assert!(
+        !body.starts_with(b"User-agent:"),
+        "a POST should reach the publisher fallback and not be given the file"
+    );
+}
+
+#[test]
+fn robots_txt_is_this_servers_only_when_the_settings_carry_the_section() {
+    let registered = |router: RouterService| -> Vec<(String, String)> {
+        router
+            .routes()
+            .into_iter()
+            .map(|route| (route.method().to_string(), route.path().to_string()))
+            .collect()
+    };
+
+    let without = registered(test_router());
+    assert!(
+        !without.iter().any(|(_, path)| path == "/robots.txt"),
+        "without the section the path is the publisher's: {without:?}"
+    );
+
+    let with = registered(router_refusing_every_crawler());
+    for method in ["GET", "HEAD"] {
+        assert!(
+            with.iter()
+                .any(|(registered, path)| registered == method && path == "/robots.txt"),
+            "{method} /robots.txt should be answered here: {with:?}"
+        );
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_response_carries_no_blanket_rule_without_the_section() {
+    let response = route(
+        test_router(),
+        robots_request("GET", "/.well-known/trusted-server.json"),
+    )
+    .await;
+
+    assert_eq!(robots_tag(&response), None);
+}

From 6d27af2182ba55006c4c0439fdead04fbd6f902e Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 9 Oct 2026 16:13:41 +0100
Subject: [PATCH 04/15] The guides describe [robots-txt]

The configuration guide gains the section, its settings, how the file is
made and what is refused when. The API reference gains the route and its
contract. The integration guide gains the contributor extension point, and
the configuration rules list `robots-txt` among the types. The example
configuration carries the section commented out.
---
 CHANGELOG.md                      |  1 +
 docs/guide/api-reference.md       | 25 ++++++++
 docs/guide/configuration-rules.md |  8 ++-
 docs/guide/configuration.md       | 94 ++++++++++++++++++++++---------
 docs/guide/integration-guide.md   | 12 +++-
 trusted-server.example.toml       | 27 +++++++--
 6 files changed, 132 insertions(+), 35 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ed71978787..6df35cae96 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -76,6 +76,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- `[robots-txt]` serves a publisher's `/robots.txt` from the settings. `modules` names what makes the file, in order. `refuse_all` and `allow_all` are Trusted Server's own, and any other name is a module that contributes rules through the new `RobotsTxtContributor` extension point, declared with `with_robots_txt_contributor`. Trusted Server writes the file, so a contributor cannot produce a malformed one, a refusal cannot be loosened, and a path in `always_allow` stays open to every crawler refused the whole site. With `refuse_all` selected every response an adapter finalizes carries `X-Robots-Tag: noindex, nofollow`. A module's answer is held in the key-value store until its refresh is due, a module that cannot answer has its last answer served, and with none held the answer is `503` and never an empty file. Without the section `/robots.txt` reaches the publisher's origin as before.
 - `ts keys rotate` and `ts keys deactivate` rotate and retire the request signing keys from an operator's machine. They write the two stores the service links as `jwks_store` and `signing_keys` through the `fastly` CLI, with the token from `FASTLY_API_TOKEN`. A private key reaches `fastly` on its standard input and never in its arguments, and a key id no platform can store is refused before anything is written. A command that cannot read the config store writes nothing, and a delete that failed part way can be run again.
 - Attestation. With `[attestation]` in the settings a deployment serves evidence, in the sense of RFC 9334, of who operates it and which build it runs, as a page and as JSON at `[attestation] endpoint`, which defaults to `/_ts/attestation`. The evidence names the host the request arrived for, the publisher, the platform service and the build, with the time and the caller's nonce, and is signed with ECDSA P-256 under a configured context. The signing keys are a schedule compiled in from the build input `TRUSTED_SERVER_ATTESTATION_KEYS`, and a build given none answers `503` at the endpoint. A build also reports the commit, the build run and the build time it was given, from `TRUSTED_SERVER_COMMIT`, `TRUSTED_SERVER_BUILD_RUN` and `SOURCE_DATE_EPOCH`. See `docs/guide/attestation.md`.
 - One Fastly service can serve several publishers. A `__KEY` selector that contains `{host}` names an app-config blob for each host, pushed with `ts config push --key <host>`, and each request reads the blob under its own host. A host with no blob is answered `421 Misdirected Request`, never from the logical store ID or from another host's blob, and a sandbox that serves several requests keeps an application for each blob, found only by that blob's key. A selector without the placeholder reads one blob as before. `scripts/config-by-host-local-test.sh` runs it under Viceroy.
diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md
index fd31cd51e5..d34eba84ae 100644
--- a/docs/guide/api-reference.md
+++ b/docs/guide/api-reference.md
@@ -56,6 +56,7 @@ authority, scheme, and client-address headers in the innermost middleware.
 | normal  | `/first-party/proxy`                   | `GET`                                                      | literal        | `always`                              | real                  | real                  | real                  | real                  |
 | normal  | `/first-party/sign`                    | `GET`, `POST`                                              | literal        | `always`                              | real                  | real                  | real                  | real                  |
 | normal  | `/health`                              | `GET`                                                      | literal        | `always`                              | real                  | real                  | —                     | real                  |
+| normal  | `/robots.txt`                          | `GET`, `HEAD`                                              | literal        | `settings.robots_txt`                 | real                  | real                  | real                  | real                  |
 | normal  | `/static/tsjs=<file>`                  | `GET`                                                      | template       | `path.starts_with(/static/tsjs=)`     | real                  | real                  | real                  | real                  |
 | normal  | `/verify-signature`                    | `POST`                                                     | literal        | `always`                              | real                  | real                  | real                  | real                  |
 | normal  | `/{*rest}`                             | `DELETE`, `GET`, `HEAD`, `OPTIONS`, `PATCH`, `POST`, `PUT` | template       | `publisher_fallback`                  | publisher fallback    | publisher fallback    | publisher fallback    | publisher fallback    |
@@ -1024,6 +1025,30 @@ credentialed or otherwise private responses are forced `private, no-store`.
 No dedicated CORS grant or rate limiter is installed. Example: if the prefix is
 `/assets/`, request `curl -I https://edge.example.com/assets/logo.svg`.
 
+### GET /robots.txt
+
+The publisher's `robots.txt`, written from the rules the modules named by
+`[robots-txt] modules` give, in that order, as
+[Configuration](/guide/configuration#robots-txt-configuration) describes.
+
+**Contract:** Auth: none. Request body: not applicable. Rate limit: none.
+Endpoint-specific CORS: none. Answered only when the settings carry
+`[robots-txt]`. Without the section the path reaches the publisher's origin, so
+a publisher keeps their own file. `HEAD` is answered with the headers of the
+`GET` and no body. Other methods reach the publisher's origin.
+
+**Response:**
+
+- `200 OK` with the file as `text/plain; charset=utf-8` and
+  `Cache-Control: public, max-age=3600`
+- `503 Service Unavailable` with `Retry-After: 600` and
+  `Cache-Control: no-store` when a selected module cannot answer and no earlier
+  answer of its is held, because an empty file or a `404` reads to a crawler as
+  permission to crawl everything
+
+With `refuse_all` selected the file refuses every crawler, and every response
+the deployment finalizes also carries `X-Robots-Tag: noindex, nofollow`.
+
 ### Publisher fallback: `/` and `/{*rest}`
 
 **Contract:** `GET`, `POST`, `HEAD`, `OPTIONS`, `PUT`, `PATCH`, and `DELETE`
diff --git a/docs/guide/configuration-rules.md b/docs/guide/configuration-rules.md
index 781a21f470..cc821a2a4b 100644
--- a/docs/guide/configuration-rules.md
+++ b/docs/guide/configuration-rules.md
@@ -10,7 +10,7 @@ request, and why the file is built this way.
 Everything Trusted Server can switch on is a module, selected the same way:
 the Edge Cookie identity, the location and device lookups, the permission
 signals, the demand sources in an auction, the ad server that picks the
-winner, and the page integrations.
+winner, the rules in `robots.txt`, and the page integrations.
 
 ```toml
 [<type>]
@@ -21,8 +21,9 @@ setting = "value"
 ```
 
 1. **The type is the job.** Each type is one top-level table, named for what
-   its modules do: `ec`, `geo`, `device`, `permission-signal`, `demand` and
-   `ad-server`, and the section of each module type, such as `cmp` or `tag`.
+   its modules do: `ec`, `geo`, `device`, `permission-signal`, `demand`,
+   `ad-server` and `robots-txt`, and the section of each module type, such as
+   `cmp` or `tag`.
 2. **The selector chooses what runs.** A type that runs one module takes
    `module`, a string. A type that runs several takes `modules`, a list.
 3. **`[<type>.<name>]` holds the settings.** A name with nothing to set
@@ -51,6 +52,7 @@ setting = "value"
 | `permission-signal` | several, in order | `modules`, a list  | `gpc`, `gpp`, `us-privacy`, `tcf`                                                                       |
 | `demand`            | several           | `modules`, a list  | `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps`                                      |
 | `ad-server`         | one               | `module`, a string | `mock`                                                                                                  |
+| `robots-txt`        | several, in order | `modules`, a list  | `refuse_all`, `allow_all`, or an integration that supplies robots.txt rules                             |
 | `cmp`               | one               | `module`, a string | `didomi`, `sourcepoint`, `osano`                                                                        |
 | `tag`               | several           | `modules`, a list  | `google-tag-manager`                                                                                    |
 | `ad-tag`            | several           | `modules`, a list  | `google`, `google.diagnostics`                                                                          |
diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md
index 7b32121ee6..8c7f5326bc 100644
--- a/docs/guide/configuration.md
+++ b/docs/guide/configuration.md
@@ -210,37 +210,38 @@ fail and the service will return its startup-error response.
 
 ## Key Sections
 
-7 of these sections select what runs, with `module` where one runs and
+10 of these sections select what runs, with `module` where one runs and
 `modules` where several run, and each gives every selected name its own
 `[<type>.<name>]` settings table, as
 [Configuration Rules](/guide/configuration-rules) describes.
 
-| Section                                                                                                           | Selects               | Purpose                                                                                     |
-| ----------------------------------------------------------------------------------------------------------------- | --------------------- | ------------------------------------------------------------------------------------------- |
-| `[ad-server]`                                                                                                     | one module            | The ad server that picks the winner                                                         |
-| `[auction]`                                                                                                       | several modules       | Auction orchestration, bidder routes, and the modules the auction runs, `prebid` among them |
-| `[cache]`                                                                                                         | nothing               | Static and rehosted asset cache policy                                                      |
-| `[consent]`                                                                                                       | nothing               | Consent interpretation, forwarding, and conflict resolution                                 |
-| `[creative_opportunities]`                                                                                        | nothing               | Server-side page ad opportunities and templates                                             |
-| `[debug]`                                                                                                         | nothing               | Explicit non-production diagnostics                                                         |
-| `[demand]`                                                                                                        | several modules       | The auction's demand sources                                                                |
-| `[device]`                                                                                                        | one module            | Device classification                                                                       |
-| `[ec]`                                                                                                            | one module            | Edge Cookie identity, persistence, and partner sync                                         |
-| `[[fetch]]`                                                                                                       | nothing               | Which page changes run on a page as it is fetched, and in what order                        |
-| `[geo]`                                                                                                           | one module            | Which module resolves location, if any                                                      |
-| `[image_optimizer]`                                                                                               | nothing               | Reusable Fastly Image Optimizer profiles                                                    |
-| `[inspect]`                                                                                                       | nothing               | What the configuration page at `/_ts/config` shows                                          |
-| `[permission-signal]`                                                                                             | several modules       | Which permission signals are acted on, in order                                             |
-| `[proxy]`                                                                                                         | several modules       | Proxy allowlist, TLS policy, asset routes, and the first-party script proxy module          |
-| `[publisher]`                                                                                                     | nothing               | Publisher domain, origin, and proxy signing key                                             |
-| `[request_signing]`                                                                                               | nothing               | Outbound Ed25519 request signing                                                            |
-| `[response_headers]`                                                                                              | nothing               | Headers added to Trusted Server responses                                                   |
-| `[rewrite]`                                                                                                       | nothing               | First-party URL rewrite exclusions                                                          |
-| `[[serve]]`                                                                                                       | nothing               | Which page changes run on each reader's copy of a page, and in what order                   |
-| `[tester_cookie]`                                                                                                 | nothing               | Optional tester-cookie endpoints                                                            |
-| `[tinybird]`                                                                                                      | nothing               | Direct Tinybird auction telemetry                                                           |
-| `[trusted_client_ip]`                                                                                             | nothing               | Authenticated front-door client-IP forwarding                                               |
-| `[<type>]`, such as `[cmp]`, `[tag]`, `[ad-tag]`, `[bot-protection]`, `[identity]`, `[audience]` or `[framework]` | one module or several | The section of a module type, selecting the modules of that type that run                   |
+| Section                                                                                                           | Selects               | Purpose                                                                                       |
+| ----------------------------------------------------------------------------------------------------------------- | --------------------- | --------------------------------------------------------------------------------------------- |
+| `[ad-server]`                                                                                                     | one module            | The ad server that picks the winner                                                           |
+| `[auction]`                                                                                                       | several modules       | Auction orchestration, bidder routes, and the modules the auction runs, `prebid` among them   |
+| `[cache]`                                                                                                         | nothing               | Static and rehosted asset cache policy                                                        |
+| `[consent]`                                                                                                       | nothing               | Consent interpretation, forwarding, and conflict resolution                                   |
+| `[creative_opportunities]`                                                                                        | nothing               | Server-side page ad opportunities and templates                                               |
+| `[debug]`                                                                                                         | nothing               | Explicit non-production diagnostics                                                           |
+| `[demand]`                                                                                                        | several modules       | The auction's demand sources                                                                  |
+| `[device]`                                                                                                        | one module            | Device classification                                                                         |
+| `[ec]`                                                                                                            | one module            | Edge Cookie identity, persistence, and partner sync                                           |
+| `[[fetch]]`                                                                                                       | nothing               | Which page changes run on a page as it is fetched, and in what order                          |
+| `[geo]`                                                                                                           | one module            | Which module resolves location, if any                                                        |
+| `[image_optimizer]`                                                                                               | nothing               | Reusable Fastly Image Optimizer profiles                                                      |
+| `[inspect]`                                                                                                       | nothing               | What the configuration page at `/_ts/config` shows                                            |
+| `[permission-signal]`                                                                                             | several modules       | Which permission signals are acted on, in order                                               |
+| `[proxy]`                                                                                                         | several modules       | Proxy allowlist, TLS policy, asset routes, and the first-party script proxy module            |
+| `[publisher]`                                                                                                     | nothing               | Publisher domain, origin, and proxy signing key                                               |
+| `[request_signing]`                                                                                               | nothing               | Outbound Ed25519 request signing                                                              |
+| `[response_headers]`                                                                                              | nothing               | Headers added to Trusted Server responses                                                     |
+| `[rewrite]`                                                                                                       | nothing               | First-party URL rewrite exclusions                                                            |
+| `[robots-txt]`                                                                                                    | several modules       | What `/robots.txt` answers, and `X-Robots-Tag` on its responses when every crawler is refused |
+| `[[serve]]`                                                                                                       | nothing               | Which page changes run on each reader's copy of a page, and in what order                     |
+| `[tester_cookie]`                                                                                                 | nothing               | Optional tester-cookie endpoints                                                              |
+| `[tinybird]`                                                                                                      | nothing               | Direct Tinybird auction telemetry                                                             |
+| `[trusted_client_ip]`                                                                                             | nothing               | Authenticated front-door client-IP forwarding                                                 |
+| `[<type>]`, such as `[cmp]`, `[tag]`, `[ad-tag]`, `[bot-protection]`, `[identity]`, `[audience]` or `[framework]` | one module or several | The section of a module type, selecting the modules of that type that run                     |
 
 ## Example: Production Setup
 
@@ -1026,6 +1027,45 @@ Edit TOML, validate, and push again for those changes.
 Custom headers may be overwritten by application logic. Standard headers (`Content-Type`, `Content-Length`) are controlled by the application.
 :::
 
+## robots.txt Configuration
+
+What `/robots.txt` answers. With no `[robots-txt]` section the publisher keeps their own file and `/robots.txt` reaches the origin like any other path.
+
+### `[robots-txt]`
+
+| Field          | Type                    | Required | Description                                                                                                                                                                                                                |
+| -------------- | ----------------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `modules`      | String or Array[String] | Yes      | What makes the file, in order. `refuse_all` refuses every crawler, `allow_all` allows every crawler everywhere, and any other name is a module that supplies robots.txt rules. A single name is the same as a list of one. |
+| `always_allow` | Array[String]           | No       | Paths kept open to every crawler refused the whole site, each starting with `/`. A narrower rule still applies, so `Disallow: /ads` closes off `/ads.txt`.                                                                 |
+| `sitemap`      | String                  | No       | Added as a `Sitemap:` line.                                                                                                                                                                                                |
+| `top_text`     | String                  | No       | The publisher's own text, placed above the rules.                                                                                                                                                                          |
+| `bottom_text`  | String                  | No       | The publisher's own text, placed below the rules.                                                                                                                                                                          |
+
+A module with settings of its own reads them from its own block, `[robots-txt.<name>]`, where `<name>` is the name `modules` selects it by. Trusted Server does not read those blocks.
+
+**How the file is made.** Modules contribute rules and Trusted Server writes the file, so these hold whatever a module returns.
+
+1. With `refuse_all` selected the file is the refusal and nothing else, whatever else is selected, and every response the deployment finalizes carries `X-Robots-Tag: noindex, nofollow`.
+2. A path in `always_allow` stays open to every crawler a module's rules refuse the whole site to, `*` included. A narrower rule is kept as the module gave it, so `Disallow: /ads` still closes off `/ads.txt`.
+3. The rules appear in the order `modules` names them, between `top_text` above and the `Sitemap` line and `bottom_text` below.
+4. A module's answer is held in the key-value store and served until the module's own refresh is due. A module that cannot answer has its last answer served past its age, for as long as the store keeps it, which is 30 days where the store can give an entry a lifetime. With no answer held at all the file is `503` with `Retry-After`, never an empty file, which a crawler would read as permission to crawl everything.
+5. Spin's key-value store gives an entry no lifetime, so there an answer is kept until it is replaced, and an answer held for settings that have since changed stays until it is deleted from the store.
+6. A deployment whose key-value store cannot be opened holds nothing, so there every request for the file asks each module.
+
+**Which responses carry the header.** An adapter writes `X-Robots-Tag` last when it finalizes a response, replacing any tag already there, so neither an origin page nor `[response_headers]` can loosen it. That covers every route, the publisher fallback and the error path. A response sent before that step does not carry it. On every adapter but Fastly that is the `404` or `405` the router gives a request it does not route. On Fastly it is the two debug endpoints under `/_ts/debug` and a failure before the settings are read.
+
+**Refused when the settings load**: `modules` absent, empty or naming one thing twice, a key that is not a setting, and a block for a module `modules` does not name. A name that nothing the deployment runs supplies is refused at startup.
+
+**A publisher who sells advertising almost certainly wants `/ads.txt` in `always_allow`.** Google's help for publishers says "The ads.txt file for a domain may be ignored by crawlers if the robots.txt file on a domain disallows one of the following", and lists first "The crawling of the URL path on which an ads.txt file is posted" ([AdSense Help](https://support.google.com/adsense/answer/7679060)). A file that refuses everything disallows that path, so a refusal without the allowance can leave the publisher's `ads.txt` unread.
+
+**Example**:
+
+```toml
+[robots-txt]
+modules = ["refuse_all"]
+always_allow = ["/ads.txt"]
+```
+
 ## Request Signing
 
 Configuration for Ed25519 request signing.
diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md
index 3cc955aaf4..d8a7cb2a0a 100644
--- a/docs/guide/integration-guide.md
+++ b/docs/guide/integration-guide.md
@@ -315,10 +315,11 @@ builder every integration uses.
 | `.with_ec_module(name, ...)`                          | Offers an Edge Cookie module that `[ec] module` may select by that name                         |
 | `.with_geo_module(name, ...)`                         | Offers a geo module that `[geo] module` may select by that name                                 |
 | `.with_device_module(name, ...)`                      | Offers a device module that `[device] module` may select by that name                           |
+| `.with_robots_txt_contributor(name, ...)`             | Offers rules for `/robots.txt` that `[robots-txt] modules` may select by that name              |
 
 The three script delivery choices are exclusive and the last call wins.
 
-Each of the last three takes the module's name, which is the path under
+Each of the last four takes the module's name, which is the path under
 `crates/` of the crate the module lives in, as `module_name!()` gives it.
 `[ec] module`, `[geo] module` and `[device] module` read a written name the
 way a section does, as written or with the type folder (`edgecookie`, `geo`
@@ -331,6 +332,15 @@ An Edge Cookie module is declared under the name its own `id` returns.
 Startup refuses a registration where the two differ, and two registrations
 that supply a module of one type under one name.
 
+A robots.txt contributor implements `RobotsTxtContributor`. It returns groups
+of records and never bytes, so it cannot produce a malformed file. Trusted
+Server asks it for the request without its query, holds the answer until the
+contributor's `refresh` is due, and writes the file from every contributor
+`[robots-txt] modules` names, in that order. `fingerprint` names the settings
+an answer was given for, so an answer held for other settings is never
+served. Trusted Server hashes it into the key the answer is held under, so
+nothing of it is written to the store.
+
 The Fastly adapter is the one adapter that classifies a request, so a device
 module is asked there and on no other adapter. It is shown the User-Agent and
 the request's cookies.
diff --git a/trusted-server.example.toml b/trusted-server.example.toml
index f0a6727307..5b5101ed84 100644
--- a/trusted-server.example.toml
+++ b/trusted-server.example.toml
@@ -27,10 +27,10 @@
 #   [<type>.<name>]       # only when that name has settings
 #   setting = "value"
 #
-# The types are ec, geo, device, permission-signal, demand, ad-server and
-# integration. A settings table its type's selector does not name refuses
-# startup, so there is no stale block to misread. The complete rules, and what
-# is checked when, are in docs/guide/configuration-rules.md.
+# The types are ec, geo, device, permission-signal, demand, ad-server,
+# robots-txt and integration. A settings table its type's selector does not
+# name refuses startup, so there is no stale block to misread. The complete
+# rules, and what is checked when, are in docs/guide/configuration-rules.md.
 #
 # `TRUSTED_SERVER__` env vars can override values when the `ts` CLI builds and
 # validates this config for push (e.g. TRUSTED_SERVER__PUBLISHER__DOMAIN=...;
@@ -185,6 +185,25 @@ pull_sync_concurrency = 3
 # [response_headers]
 # X-Robots-Tag = "noindex"
 
+# What /robots.txt answers. With this section absent the publisher keeps
+# their own file and /robots.txt reaches their origin like any other path.
+#
+# `modules` names what makes the file, in order, as one name or a list.
+# refuse_all refuses every crawler and puts X-Robots-Tag: noindex, nofollow on
+# the responses it sends, allow_all allows every crawler everywhere, and any
+# other name is a module that supplies rules. Modules contribute rules and
+# Trusted Server writes the file. A module that cannot answer has its last
+# answer served past its age, and with nothing held the answer is 503, never
+# an empty file, which a crawler would read as permission.
+#
+# A module's own settings go in its own block, [robots-txt.<name>].
+# A publisher who sells advertising wants /ads.txt in always_allow, because
+# a crawler may ignore an ads.txt that robots.txt disallows.
+# [robots-txt]
+# modules = ["refuse_all"]
+# always_allow = ["/ads.txt"]
+# sitemap = "https://publisher.example.com/sitemap.xml"
+
 # Sign outbound OpenRTB/API requests. The runtime reads its keys from fixed
 # store names (`jwks_store` / `signing_keys`), and `ts keys` writes them. To keep
 # signing OFF, leave this whole block commented out.

From e6a1d5108ce49bed323f21a59d896563e3e64727 Mon Sep 17 00:00:00 2001
From: James Rogers <JamesR@51degrees.com>
Date: Mon, 12 Oct 2026 15:20:46 +0100
Subject: [PATCH 05/15] A robots.txt value cannot start a record of its own

Core takes CR and LF out of every value, comment, record name and user agent it writes, so a module cannot open the site by smuggling a second group. Disallow written with wildcards that match every path, such as Disallow: /*, now refuses the whole site, so always_allow applies to it. Each rationale is stated once and the comments that repeated it are cut.
---
 crates/trusted-server-adapter-axum/src/app.rs | 14 +--
 .../src/app.rs                                |  5 +-
 .../trusted-server-adapter-fastly/src/app.rs  |  7 +-
 crates/trusted-server-adapter-spin/src/app.rs |  5 +-
 .../src/robots_txt/contributor.rs             |  9 +-
 .../src/robots_txt/document.rs                | 89 ++++++++++++++-----
 .../src/robots_txt/holding.rs                 |  4 +-
 .../trusted-server-core/src/robots_txt/mod.rs | 25 +-----
 docs/guide/configuration-rules.md             |  2 +-
 trusted-server.example.toml                   |  2 -
 10 files changed, 81 insertions(+), 81 deletions(-)

diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs
index c63f62b8d8..2634ef5f4c 100644
--- a/crates/trusted-server-adapter-axum/src/app.rs
+++ b/crates/trusted-server-adapter-axum/src/app.rs
@@ -283,10 +283,9 @@ where
     let services = state.services_for_request(&ctx);
     let mut req = ctx.into_request();
     // The single preparation point for the adapter. Every route in the table
-    // except `/health` and the attestation pages is registered to
-    // `named_route_handler` or `fallback_handler`, and both wrap this
-    // function, so each request has its modules' preparers run exactly once
-    // and always before routing.
+    // except `/health` is registered to `named_route_handler` or
+    // `fallback_handler`, and both wrap this function, so each request has its
+    // modules' preparers run exactly once and always before routing.
     if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) {
         return Ok(http_error(&error));
     }
@@ -538,8 +537,7 @@ fn named_routes() -> [NamedRoute; 15] {
             primary_methods: &[Method::GET, Method::POST],
             handler: NamedRouteHandler::FirstPartyProxyRebuild,
         },
-        // The publisher's robots.txt, served only when the settings carry a
-        // `[robots-txt]` section, see `named_route_is_served`.
+        // Only when the settings carry `[robots-txt]`.
         NamedRoute {
             path: ROBOTS_TXT_PATH,
             primary_methods: &[Method::GET, Method::HEAD],
@@ -549,10 +547,6 @@ fn named_routes() -> [NamedRoute; 15] {
 }
 
 /// Whether a named route is served for these settings.
-///
-/// `/robots.txt` is this server's only when the publisher's settings carry a
-/// `[robots-txt]` section. Without one the path reaches the publisher's
-/// origin like any other, so a publisher keeping their own file is left alone.
 fn named_route_is_served(settings: &Settings, handler: NamedRouteHandler) -> bool {
     match handler {
         NamedRouteHandler::RobotsTxt => settings.robots_txt.is_some(),
diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs
index 1a603d42af..f4af6046d3 100644
--- a/crates/trusted-server-adapter-cloudflare/src/app.rs
+++ b/crates/trusted-server-adapter-cloudflare/src/app.rs
@@ -889,10 +889,7 @@ fn build_router(state: &Arc<AppState>) -> RouterService {
             }
         }
 
-        // `/robots.txt` is this server's only when the settings carry a
-        // `[robots-txt]` section. Without one the path reaches the publisher's
-        // origin like any other, so a publisher keeping their own file is
-        // left alone.
+        // Only when the settings carry `[robots-txt]`.
         if state.settings.robots_txt.is_some() {
             let robots_txt = make_handler(Arc::clone(&state), |s, services, req| async move {
                 handle_robots_txt(&s.settings, &services, &s.registry, req).await
diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs
index 6e2a4073e4..ab422e850c 100644
--- a/crates/trusted-server-adapter-fastly/src/app.rs
+++ b/crates/trusted-server-adapter-fastly/src/app.rs
@@ -1413,8 +1413,7 @@ const NAMED_ROUTES: &[NamedRoute] = &[
         primary_methods: &[Method::GET, Method::POST],
         handler: NamedRouteHandler::FirstPartyProxyRebuild,
     },
-    // The publisher's robots.txt, served only when the settings carry a
-    // `[robots-txt]` section, see `named_route_is_served`.
+    // Only when the settings carry `[robots-txt]`.
     NamedRoute {
         path: ROBOTS_TXT_PATH,
         primary_methods: &[Method::GET, Method::HEAD],
@@ -1423,10 +1422,6 @@ const NAMED_ROUTES: &[NamedRoute] = &[
 ];
 
 /// Whether a named route is served for these settings.
-///
-/// `/robots.txt` is this server's only when the publisher's settings carry a
-/// `[robots-txt]` section. Without one the path reaches the publisher's
-/// origin like any other, so a publisher keeping their own file is left alone.
 fn named_route_is_served(settings: &Settings, handler: NamedRouteHandler) -> bool {
     match handler {
         NamedRouteHandler::RobotsTxt => settings.robots_txt.is_some(),
diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs
index 57b2c31154..0c56c5b69b 100644
--- a/crates/trusted-server-adapter-spin/src/app.rs
+++ b/crates/trusted-server-adapter-spin/src/app.rs
@@ -1104,10 +1104,7 @@ fn build_router(state: &Arc<AppState>) -> RouterService {
             }
         }
 
-        // `/robots.txt` is this server's only when the settings carry a
-        // `[robots-txt]` section. Without one the path reaches the publisher's
-        // origin like any other, so a publisher keeping their own file is
-        // left alone.
+        // Only when the settings carry `[robots-txt]`.
         if state.settings.robots_txt.is_some() {
             let s = Arc::clone(&state);
             let robots_txt_handler = move |ctx: RequestContext| {
diff --git a/crates/trusted-server-core/src/robots_txt/contributor.rs b/crates/trusted-server-core/src/robots_txt/contributor.rs
index 530a1ad82d..c4ee61a4ea 100644
--- a/crates/trusted-server-core/src/robots_txt/contributor.rs
+++ b/crates/trusted-server-core/src/robots_txt/contributor.rs
@@ -12,9 +12,8 @@ use crate::permissions::PermissionSet;
 /// The `[robots-txt] modules` entry that refuses every crawler.
 ///
 /// Core's own, and it names no vendor. Selecting it makes the file the refusal
-/// and nothing else, whatever else is selected, because a refusal cannot be
-/// loosened, and it puts `X-Robots-Tag: noindex, nofollow` on every response an
-/// adapter finalizes.
+/// and nothing else, whatever else is selected, and puts
+/// [`REFUSE_ALL_ROBOTS_TAG`](super::REFUSE_ALL_ROBOTS_TAG) on every response.
 pub const REFUSE_ALL: &str = "refuse_all";
 
 /// The `[robots-txt] modules` entry that allows every crawler everywhere.
@@ -71,9 +70,7 @@ pub trait RobotsTxtContributor: Send + Sync + core::fmt::Debug {
     ///
     /// A contribution with no groups is an answer, and is served as one. An
     /// error is a failure, and core serves this contributor's held copy
-    /// instead. With no held copy the whole file fails closed with `503`,
-    /// because leaving out a contributor's rules would allow by omission
-    /// whatever those rules refused.
+    /// instead. With no held copy the whole file fails closed with `503`.
     ///
     /// # Errors
     ///
diff --git a/crates/trusted-server-core/src/robots_txt/document.rs b/crates/trusted-server-core/src/robots_txt/document.rs
index 56d0665640..1e349d3d3a 100644
--- a/crates/trusted-server-core/src/robots_txt/document.rs
+++ b/crates/trusted-server-core/src/robots_txt/document.rs
@@ -10,14 +10,8 @@
 //!    refuses the whole site to, see [`assemble`].
 //! 3. The publisher's own text and the `Sitemap` line go where the publisher
 //!    put them, around what the contributors gave.
-//!
-//! The record names in a contribution are written in one spelling,
-//! `User-Agent`, `Allow` and `Disallow`, and everything else is written back
-//! exactly as it was read. A contributor that reads a finished file in that
-//! spelling and hands it over as a [`Contribution`] therefore changes nothing in
-//! its bytes, which is what lets another implementation agree with this one byte
-//! for byte. The [`refusal`] is written whole rather than from contributions,
-//! and spells its first record `User-agent`.
+
+use std::borrow::Cow;
 
 use serde::{Deserialize, Serialize};
 
@@ -46,15 +40,28 @@ pub enum Record {
 impl Record {
     fn line(&self) -> String {
         match self {
-            Self::Allow(path) => format!("Allow: {path}"),
-            Self::Disallow(path) => format!("Disallow: {path}"),
-            Self::Comment(text) => format!("#{text}"),
-            Self::Other { name, value } => format!("{name}: {value}"),
+            Self::Allow(path) => format!("Allow: {}", one_line(path)),
+            Self::Disallow(path) => format!("Disallow: {}", one_line(path)),
+            Self::Comment(text) => format!("#{}", one_line(text)),
+            Self::Other { name, value } => format!("{}: {}", one_line(name), one_line(value)),
         }
     }
 
+    /// `Disallow: /`, or the same rule written with wildcards, such as
+    /// `Disallow: /*`, which RFC 9309 section 2.2.3 reads as matching every path.
     fn refuses_the_whole_site(&self) -> bool {
-        matches!(self, Self::Disallow(path) if path == "/")
+        matches!(self, Self::Disallow(path)
+            if one_line(path).strip_prefix('/').is_some_and(|rest| rest.bytes().all(|b| b == b'*')))
+    }
+}
+
+/// `value` with every CR and LF taken out, so that it cannot end its line and
+/// start a record of its own.
+fn one_line(value: &str) -> Cow<'_, str> {
+    if value.contains(['\r', '\n']) {
+        Cow::Owned(value.replace(['\r', '\n'], ""))
+    } else {
+        Cow::Borrowed(value)
     }
 }
 
@@ -162,7 +169,7 @@ impl Contribution {
             let mut lines: Vec<String> = group
                 .user_agents
                 .iter()
-                .map(|agent| format!("User-Agent: {agent}"))
+                .map(|agent| format!("User-Agent: {}", one_line(agent)))
                 .collect();
             lines.extend(group.records.iter().map(Record::line));
             parts.push(lines.join("\n"));
@@ -181,8 +188,7 @@ fn push(contribution: &mut Contribution, current: &mut Option<Group>, record: Re
 /// What `allow_all` contributes, which is every crawler allowed everywhere.
 ///
 /// Permission the document says, rather than permission that follows from a
-/// lookup coming back empty. The two produce the same bytes, and only one of
-/// them was meant.
+/// lookup coming back empty.
 #[must_use]
 pub fn allow_all() -> Contribution {
     Contribution {
@@ -201,13 +207,6 @@ pub fn allow_all() -> Contribution {
 /// added, because a refusal cannot be loosened. `Allow` beats `Disallow` on
 /// the longest match under RFC 9309 section 2.2.2, so each allowance lets that
 /// one path through.
-///
-/// A publisher selling advertising should allow `/ads.txt`, because the
-/// advertising platforms may ignore an `ads.txt` that robots.txt disallows,
-/// and a blanket `Disallow: /` is exactly that case. It is not added by
-/// default, because a domain with no `ads.txt` would be inviting crawlers to
-/// a path it does not have. The configuration guide quotes one platform's own
-/// statement of this.
 #[must_use]
 pub fn refusal(always_allow: &[String]) -> String {
     let mut file = String::from("User-agent: *");
@@ -291,7 +290,7 @@ fn kept_open(contributions: &[Contribution], paths: &[String]) -> Option<String>
     }
     let mut lines: Vec<String> = names
         .iter()
-        .map(|name| format!("User-Agent: {name}"))
+        .map(|name| format!("User-Agent: {}", one_line(name)))
         .collect();
     lines.extend(paths.iter().map(|path| format!("Allow: {path}")));
     Some(lines.join("\n"))
@@ -416,6 +415,48 @@ mod tests {
         );
     }
 
+    #[test]
+    fn a_wildcard_that_matches_every_path_refuses_the_whole_site() {
+        let mut settings = config();
+        settings.always_allow = vec!["/ai-notice/".to_owned()];
+
+        assert_eq!(
+            assemble(
+                &settings,
+                &[Contribution::parse("User-Agent: ExampleBot\nDisallow: /*")]
+            ),
+            "User-Agent: ExampleBot\nDisallow: /*\n\n\
+             User-Agent: ExampleBot\nAllow: /ai-notice/\n",
+            "`Disallow: /*` is the same rule as `Disallow: /`"
+        );
+    }
+
+    #[test]
+    fn a_line_break_in_a_value_cannot_start_a_record() {
+        let contribution = Contribution {
+            preamble: vec![Record::Comment(" note\nAllow: /".to_owned())],
+            groups: vec![Group {
+                user_agents: vec!["ExampleBot\r\nUser-Agent: *".to_owned()],
+                records: vec![
+                    Record::Disallow("/x\nUser-Agent: *\nAllow: /".to_owned()),
+                    Record::Other {
+                        name: "TDL\r".to_owned(),
+                        value: "a\rAllow: /".to_owned(),
+                    },
+                ],
+            }],
+        };
+
+        assert_eq!(
+            assemble(&config(), &[contribution]),
+            "# noteAllow: /\n\n\
+             User-Agent: ExampleBotUser-Agent: *\n\
+             Disallow: /xUser-Agent: *Allow: /\n\
+             TDL: aAllow: /\n",
+            "every value stays on the line its record starts"
+        );
+    }
+
     #[test]
     fn nothing_is_kept_open_when_nothing_is_refused() {
         let mut settings = config();
diff --git a/crates/trusted-server-core/src/robots_txt/holding.rs b/crates/trusted-server-core/src/robots_txt/holding.rs
index e81c78b56b..4a62f6b5e7 100644
--- a/crates/trusted-server-core/src/robots_txt/holding.rs
+++ b/crates/trusted-server-core/src/robots_txt/holding.rs
@@ -6,9 +6,7 @@
 //!
 //! What is held is kept far longer than any refresh, and is served past its
 //! age when the contributor fails, because yesterday's rules are a better
-//! answer than none. A `404` or an empty file reads to a crawler as permission
-//! to crawl everything, which is the opposite of what a publisher who refused
-//! anything asked for.
+//! answer than none.
 //!
 //! A store that cannot give an entry a lifetime is written without one. An
 //! answer held for settings that have since changed then stays in that store,
diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs
index 86d0f3b77b..7f27ca1a4e 100644
--- a/crates/trusted-server-core/src/robots_txt/mod.rs
+++ b/crates/trusted-server-core/src/robots_txt/mod.rs
@@ -104,9 +104,6 @@ pub struct RobotsTxtConfig {
     /// refused crawler is sent to for example. A narrower rule, such as
     /// `Disallow: /ads`, still closes off `/ads.txt`. `/robots.txt` needs no
     /// entry, because RFC 9309 section 2.2.2 says it is implicitly allowed.
-    ///
-    /// A publisher who sells advertising almost certainly wants `/ads.txt`
-    /// here, for the reason given on [`document::refusal`].
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     #[validate(custom(function = validate_always_allow))]
     pub always_allow: Vec<String>,
@@ -265,20 +262,9 @@ fn configuration(message: String) -> TrustedServerError {
     TrustedServerError::Configuration { message }
 }
 
-/// Applies the publisher's blanket crawler rule to an outgoing response.
-///
-/// Called once where an adapter finalizes a response, rather than on each
-/// route, so a route added later cannot be the one page a crawler is allowed
-/// to index. A site taken out of search has to be taken out of search
-/// entirely.
-///
-/// Replaces rather than appends. Selecting `refuse_all` means the whole site is
-/// not to be indexed, so a tag an origin page set for its own reasons must not
-/// be able to weaken it, and two tags on one response is how a crawler is given
-/// a choice.
-///
-/// Does nothing unless `refuse_all` is selected, because a file built from
-/// contributors already says what each crawler may do.
+/// Puts [`REFUSE_ALL_ROBOTS_TAG`] on an outgoing response when `refuse_all` is
+/// selected, in place of any `X-Robots-Tag` it carried, and does nothing
+/// otherwise. Called once where an adapter finalizes a response.
 pub fn apply_response_robots_tag(settings: &Settings, response: &mut Response<EdgeBody>) {
     if settings
         .robots_txt
@@ -342,8 +328,7 @@ async fn answer(
             continue;
         }
         let Some(contributor) = registry.robots_txt_contributor(id) else {
-            // The registry refuses a selection nothing supplies when it is
-            // built, so this is a wiring fault, and it still fails closed.
+            // Unreachable, the registry refuses this name when it is built.
             return Ok(respond(Err(Report::new(configuration(format!(
                 "[robots-txt] modules names `{id}`, which no module in this deployment supplies"
             ))))));
@@ -632,8 +617,6 @@ mod tests {
         );
     }
 
-    /// Leaving out the rules of a contributor that failed would allow by
-    /// omission what those rules refused, so the whole file fails closed.
     #[tokio::test]
     async fn with_nothing_held_a_failure_is_service_unavailable() {
         let failing: Arc<dyn RobotsTxtContributor> =
diff --git a/docs/guide/configuration-rules.md b/docs/guide/configuration-rules.md
index cc821a2a4b..fd4ac994f5 100644
--- a/docs/guide/configuration-rules.md
+++ b/docs/guide/configuration-rules.md
@@ -52,7 +52,7 @@ setting = "value"
 | `permission-signal` | several, in order | `modules`, a list  | `gpc`, `gpp`, `us-privacy`, `tcf`                                                                       |
 | `demand`            | several           | `modules`, a list  | `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps`                                      |
 | `ad-server`         | one               | `module`, a string | `mock`                                                                                                  |
-| `robots-txt`        | several, in order | `modules`, a list  | `refuse_all`, `allow_all`, or an integration that supplies robots.txt rules                             |
+| `robots-txt`        | several, in order | `modules`, a list  | `refuse_all`, `allow_all`, or a module that supplies robots.txt rules                                   |
 | `cmp`               | one               | `module`, a string | `didomi`, `sourcepoint`, `osano`                                                                        |
 | `tag`               | several           | `modules`, a list  | `google-tag-manager`                                                                                    |
 | `ad-tag`            | several           | `modules`, a list  | `google`, `google.diagnostics`                                                                          |
diff --git a/trusted-server.example.toml b/trusted-server.example.toml
index 5b5101ed84..cbec9192b3 100644
--- a/trusted-server.example.toml
+++ b/trusted-server.example.toml
@@ -197,8 +197,6 @@ pull_sync_concurrency = 3
 # an empty file, which a crawler would read as permission.
 #
 # A module's own settings go in its own block, [robots-txt.<name>].
-# A publisher who sells advertising wants /ads.txt in always_allow, because
-# a crawler may ignore an ads.txt that robots.txt disallows.
 # [robots-txt]
 # modules = ["refuse_all"]
 # always_allow = ["/ads.txt"]

From 73fd38db3967a8856fe152afd4f01680ed5c785a Mon Sep 17 00:00:00 2001
From: James Rogers <JamesR@51degrees.com>
Date: Mon, 12 Oct 2026 17:38:41 +0100
Subject: [PATCH 06/15] A robots.txt comment ends neither a rule nor a run of
 User-Agent lines

A rule followed by a comment, such as "Disallow: / # closed", kept the
comment in its path, so it was not read as refusing the whole site and an
always_allow path was not kept open. A comment line between two User-Agent
lines also started a new group. A comment is now kept as a record of its
own after the rule it followed, and only a rule ends a run of User-Agent
lines.
---
 .../src/robots_txt/document.rs                | 114 ++++++++++++------
 1 file changed, 76 insertions(+), 38 deletions(-)

diff --git a/crates/trusted-server-core/src/robots_txt/document.rs b/crates/trusted-server-core/src/robots_txt/document.rs
index 1e349d3d3a..8973e61595 100644
--- a/crates/trusted-server-core/src/robots_txt/document.rs
+++ b/crates/trusted-server-core/src/robots_txt/document.rs
@@ -88,56 +88,58 @@ impl Contribution {
     /// A contribution read from `robots.txt` text, keeping every group, record
     /// and comment in the order it came.
     ///
-    /// A `User-Agent` line after a group's records starts the next group, and
+    /// A `User-Agent` line after a group's rules starts the next group, and
     /// consecutive `User-Agent` lines name the crawlers of one group, as RFC
-    /// 9309 section 2.1 describes. Blank lines and lines with no `:` carry
-    /// nothing and are passed over. A record before the first group is kept in
-    /// the preamble rather than dropped, so nothing given is lost.
+    /// 9309 section 2.1 describes. A `#` starts a comment wherever it is, as
+    /// section 2.2 allows, and the comment is kept as a record of its own after
+    /// the rule it followed. Blank lines and lines with no `:` carry nothing
+    /// and are passed over. A record before the first group is kept in the
+    /// preamble rather than dropped, so nothing given is lost.
     #[must_use]
     pub fn parse(text: &str) -> Self {
         let mut contribution = Self::default();
         let mut current: Option<Group> = None;
         for raw in text.split('\n') {
-            let line = raw.trim();
-            if line.is_empty() {
-                continue;
+            let (rule, comment) = match raw.split_once('#') {
+                Some((rule, comment)) => (rule.trim(), Some(comment.trim_end())),
+                None => (raw.trim(), None),
+            };
+            if let Some((name, value)) = rule.split_once(':') {
+                let (name, value) = (name.trim(), value.trim().to_owned());
+                match name.to_ascii_lowercase().as_str() {
+                    "user-agent" => {
+                        if current.as_ref().is_some_and(|group| {
+                            group
+                                .records
+                                .iter()
+                                .any(|record| !matches!(record, Record::Comment(_)))
+                        }) && let Some(finished) = current.take()
+                        {
+                            contribution.groups.push(finished);
+                        }
+                        current
+                            .get_or_insert_with(Group::default)
+                            .user_agents
+                            .push(value);
+                    }
+                    "allow" => push(&mut contribution, &mut current, Record::Allow(value)),
+                    "disallow" => push(&mut contribution, &mut current, Record::Disallow(value)),
+                    _ => push(
+                        &mut contribution,
+                        &mut current,
+                        Record::Other {
+                            name: name.to_owned(),
+                            value,
+                        },
+                    ),
+                }
             }
-            if let Some(comment) = line.strip_prefix('#') {
+            if let Some(comment) = comment {
                 push(
                     &mut contribution,
                     &mut current,
                     Record::Comment(comment.to_owned()),
                 );
-                continue;
-            }
-            let Some((name, value)) = line.split_once(':') else {
-                continue;
-            };
-            let (name, value) = (name.trim(), value.trim().to_owned());
-            match name.to_ascii_lowercase().as_str() {
-                "user-agent" => {
-                    if current
-                        .as_ref()
-                        .is_some_and(|group| !group.records.is_empty())
-                        && let Some(finished) = current.take()
-                    {
-                        contribution.groups.push(finished);
-                    }
-                    current
-                        .get_or_insert_with(Group::default)
-                        .user_agents
-                        .push(value);
-                }
-                "allow" => push(&mut contribution, &mut current, Record::Allow(value)),
-                "disallow" => push(&mut contribution, &mut current, Record::Disallow(value)),
-                _ => push(
-                    &mut contribution,
-                    &mut current,
-                    Record::Other {
-                        name: name.to_owned(),
-                        value,
-                    },
-                ),
             }
         }
         if let Some(group) = current {
@@ -431,6 +433,42 @@ mod tests {
         );
     }
 
+    #[test]
+    fn a_comment_after_a_rule_does_not_hide_a_whole_site_refusal() {
+        let mut settings = config();
+        settings.always_allow = vec!["/ads.txt".to_owned()];
+        let parsed = Contribution::parse("User-Agent: *\nDisallow: / # temporarily closed");
+
+        assert_eq!(
+            parsed.groups[0].records,
+            vec![
+                Record::Disallow("/".to_owned()),
+                Record::Comment(" temporarily closed".to_owned()),
+            ]
+        );
+        assert!(
+            assemble(&settings, &[parsed]).ends_with("\n\nUser-Agent: *\nAllow: /ads.txt\n"),
+            "the path stays open to the crawlers refused the site"
+        );
+    }
+
+    #[test]
+    fn a_comment_between_user_agent_lines_keeps_their_group() {
+        let mut settings = config();
+        settings.always_allow = vec!["/ads.txt".to_owned()];
+        let parsed = Contribution::parse(
+            "User-Agent: ExampleBot\n# the same rules for its sibling\nUser-Agent: OtherBot\nDisallow: /",
+        );
+
+        assert_eq!(parsed.groups.len(), 1);
+        assert_eq!(parsed.groups[0].user_agents, vec!["ExampleBot", "OtherBot"]);
+        assert!(
+            assemble(&settings, &[parsed])
+                .ends_with("\n\nUser-Agent: ExampleBot\nUser-Agent: OtherBot\nAllow: /ads.txt\n"),
+            "every crawler of the group is kept open to the path"
+        );
+    }
+
     #[test]
     fn a_line_break_in_a_value_cannot_start_a_record() {
         let contribution = Contribution {

From 141ae9eb89f59cad93d05379f97d4ebb8de78ee8 Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 16 Oct 2026 11:16:32 +0100
Subject: [PATCH 07/15] Hold a robots.txt answer for the host the request was
 addressed to

The key an answer was held under named the publisher, the contributor and
its fingerprint, while the contributor was asked for the host read from
Forwarded, X-Forwarded-Host or Host. A request with a forwarded host, which
Axum and Cloudflare leave in place, decided the answer every crawler of the
publisher was then served, and one host of a deployment was served the
answer made for another.

The contributor is now asked for the host the request was addressed to,
read from Host and never from a forwarded header, and that host is part of
the key. The reading moves from the attestation module to http_util, so
the signed host and the held key read the host the same way.
---
 crates/trusted-server-core/src/attestation.rs |  24 +---
 crates/trusted-server-core/src/http_util.rs   |  20 ++++
 .../trusted-server-core/src/module_context.rs |   9 ++
 .../src/robots_txt/contributor.rs             |  16 +--
 .../src/robots_txt/holding.rs                 |  78 ++++++++-----
 .../trusted-server-core/src/robots_txt/mod.rs | 106 +++++++++++++++++-
 6 files changed, 198 insertions(+), 55 deletions(-)

diff --git a/crates/trusted-server-core/src/attestation.rs b/crates/trusted-server-core/src/attestation.rs
index 56bcaa4f8d..3e2af66499 100644
--- a/crates/trusted-server-core/src/attestation.rs
+++ b/crates/trusted-server-core/src/attestation.rs
@@ -23,6 +23,7 @@ use serde::{Deserialize, Serialize};
 use serde_json::{Value, json};
 use validator::{Validate, ValidationError};
 
+use crate::http_util::addressed_host;
 use crate::inspect::config::{CONFIG_JSON_PATH, CONFIG_PAGE_PATH};
 use crate::inspect::data::DATA_PAGE_PATH;
 use crate::inspect::html_escape;
@@ -356,7 +357,7 @@ fn answer(
             "No signing key is built in for this time, so no evidence can be signed.",
         );
     };
-    let host = request_host(req);
+    let host = addressed_host(req);
     let evidence = AttestationEvidence {
         v: EVIDENCE_VERSION,
         operator: &config.operator,
@@ -420,7 +421,7 @@ fn answer(
 /// request's scheme and the host it was addressed to.
 ///
 /// The scheme follows a `Forwarded` or `X-Forwarded-Proto` header when one is
-/// sent. The host is read the way [`request_host`] reads the one the evidence
+/// sent. The host is read the way [`addressed_host`] reads the one the evidence
 /// signs, so the link and the signature name the same host, and a forwarded
 /// host header is never read.
 fn json_address(req: &Request<EdgeBody>) -> String {
@@ -511,21 +512,6 @@ fn nonce_from_query(query: Option<&str>) -> Result<Option<String>, &'static str>
     }
 }
 
-/// The host the request was addressed to, lowercased with any port removed.
-fn request_host(req: &Request<EdgeBody>) -> String {
-    let raw = req
-        .headers()
-        .get(header::HOST)
-        .and_then(|value| value.to_str().ok())
-        .or_else(|| req.uri().host())
-        .unwrap_or_default();
-    let without_port = match raw.rsplit_once(':') {
-        Some((host, port)) if port.bytes().all(|b| b.is_ascii_digit()) => host,
-        _ => raw,
-    };
-    without_port.to_ascii_lowercase()
-}
-
 /// Seconds since the epoch as RFC 3339 in UTC, to the second.
 fn rfc3339(seconds: u64) -> String {
     i64::try_from(seconds)
@@ -1806,13 +1792,13 @@ mod tests {
             .uri("https://Test-Publisher.com:8443/_ex/attestation.json")
             .body(EdgeBody::empty())
             .expect("request");
-        assert_eq!(request_host(&absolute), "test-publisher.com");
+        assert_eq!(addressed_host(&absolute), "test-publisher.com");
         let literal = Request::builder()
             .uri("/_ex/attestation.json")
             .header(header::HOST, "[::1]:8080")
             .body(EdgeBody::empty())
             .expect("request");
-        assert_eq!(request_host(&literal), "[::1]");
+        assert_eq!(addressed_host(&literal), "[::1]");
     }
 
     /// Changing a signed byte, here the host, stops the signature verifying,
diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs
index ee95183600..f2c528045f 100644
--- a/crates/trusted-server-core/src/http_util.rs
+++ b/crates/trusted-server-core/src/http_util.rs
@@ -165,6 +165,26 @@ pub(crate) fn request_host(req: &Request<EdgeBody>) -> &str {
     extract_request_host(req)
 }
 
+/// The host a request was addressed to, lowercased with any port removed.
+///
+/// Read from `Host`, or from the URI's host when there is no `Host` header,
+/// and never from `Forwarded` or `X-Forwarded-Host`, which a client can set to
+/// any host it likes. What is signed for a host, or held for everyone who
+/// asks it, is keyed on this, so one request cannot claim another host.
+pub(crate) fn addressed_host(req: &Request<EdgeBody>) -> String {
+    let raw = req
+        .headers()
+        .get(header::HOST)
+        .and_then(|value| value.to_str().ok())
+        .or_else(|| req.uri().host())
+        .unwrap_or_default();
+    let without_port = match raw.rsplit_once(':') {
+        Some((host, port)) if port.bytes().all(|b| b.is_ascii_digit()) => host,
+        _ => raw,
+    };
+    without_port.to_ascii_lowercase()
+}
+
 /// The scheme a request was made over, read as
 /// [`RequestInfo::from_request`] reads it.
 pub(crate) fn request_scheme(req: &Request<EdgeBody>, client_info: &ClientInfo) -> &'static str {
diff --git a/crates/trusted-server-core/src/module_context.rs b/crates/trusted-server-core/src/module_context.rs
index cbf8591cea..d056a8dfab 100644
--- a/crates/trusted-server-core/src/module_context.rs
+++ b/crates/trusted-server-core/src/module_context.rs
@@ -174,6 +174,15 @@ impl ResolvedRequest {
         self
     }
 
+    /// The same request for `host`, for an answer that is held under the host
+    /// it is made for, which must be the host the caller read and not one a
+    /// forwarded header names.
+    #[must_use]
+    pub(crate) fn with_host(mut self, host: String) -> Self {
+        self.host = host;
+        self
+    }
+
     /// The borrowed view a context carries.
     #[must_use]
     pub fn view(&self) -> ModuleRequest<'_> {
diff --git a/crates/trusted-server-core/src/robots_txt/contributor.rs b/crates/trusted-server-core/src/robots_txt/contributor.rs
index c4ee61a4ea..39eacb1ca4 100644
--- a/crates/trusted-server-core/src/robots_txt/contributor.rs
+++ b/crates/trusted-server-core/src/robots_txt/contributor.rs
@@ -60,13 +60,15 @@ pub trait RobotsTxtContributor: Send + Sync + core::fmt::Debug {
 
     /// This contributor's rules.
     ///
-    /// Core holds the answer for the publisher and serves it to every
-    /// crawler. So the call carries the request the answer was asked for,
-    /// being its method, path, host and scheme without its query, and the
-    /// request's services, and nothing about who asked. An implementation
-    /// hands its own function to [`ModuleCall::inject`], naming what it
-    /// needs. Its answer must be the same for every request the publisher
-    /// receives, because it is served for all of them.
+    /// Core holds the answer for the publisher and the host the request was
+    /// addressed to, and serves it to every crawler of that host. So the call
+    /// carries the request the answer was asked for, being its method, path,
+    /// host and scheme without its query, with the host never taken from a
+    /// forwarded header, and the request's services, and nothing about who
+    /// asked. An implementation hands its own function to
+    /// [`ModuleCall::inject`], naming what it needs. Its answer must be the
+    /// same for every request for that host, because it is served for all of
+    /// them.
     ///
     /// A contribution with no groups is an answer, and is served as one. An
     /// error is a failure, and core serves this contributor's held copy
diff --git a/crates/trusted-server-core/src/robots_txt/holding.rs b/crates/trusted-server-core/src/robots_txt/holding.rs
index 4a62f6b5e7..0cf5d1a679 100644
--- a/crates/trusted-server-core/src/robots_txt/holding.rs
+++ b/crates/trusted-server-core/src/robots_txt/holding.rs
@@ -8,6 +8,10 @@
 //! age when the contributor fails, because yesterday's rules are a better
 //! answer than none.
 //!
+//! An answer is held for the host the contributor was asked for, as well as
+//! for the publisher and the contributor's settings, so one host's answer is
+//! never served for another.
+//!
 //! A store that cannot give an entry a lifetime is written without one. An
 //! answer held for settings that have since changed then stays in that store,
 //! one small entry for each change, until it is deleted there.
@@ -71,7 +75,7 @@ pub(crate) async fn contribution_for(
     services: &RuntimeServices,
     now: u64,
 ) -> Result<Contribution, Report<TrustedServerError>> {
-    let key = held_key(holder, &contributor.fingerprint());
+    let key = held_key(holder, request.view().host(), &contributor.fingerprint());
     let held = held_answer(services, &key).await;
     if let Some(held) = &held
         && now.saturating_sub(held.given_at) < contributor.refresh().as_secs()
@@ -109,14 +113,14 @@ pub(crate) async fn contribution_for(
     }
 }
 
-/// The key one contributor's answer for one publisher and one fingerprint is
-/// held under. A hash, so that the fingerprint's contents, whatever they are,
-/// never appear in the store.
-fn held_key(holder: &Holder<'_>, fingerprint: &str) -> String {
+/// The key one contributor's answer for one publisher, one host and one
+/// fingerprint is held under. A hash, so that the fingerprint's contents,
+/// whatever they are, never appear in the store.
+fn held_key(holder: &Holder<'_>, host: &str, fingerprint: &str) -> String {
     let mut hasher = Sha256::new();
-    for part in [holder.publisher, holder.id, fingerprint] {
-        // Each part with its length, so that no two different triples run
-        // together into the same bytes.
+    for part in [holder.publisher, host, holder.id, fingerprint] {
+        // Each part with its length, so that no two different sets of parts
+        // run together into the same bytes.
         hasher.update((part.len() as u64).to_be_bytes());
         hasher.update(part.as_bytes());
     }
@@ -548,33 +552,57 @@ pub(crate) mod tests {
 
     #[test]
     fn a_key_holds_nothing_of_what_it_was_made_from() {
-        let key = held_key(&holder("a_contributor"), "a secret an implementer left in");
+        let key = held_key(
+            &holder("a_contributor"),
+            "crawled.example",
+            "a secret an implementer left in",
+        );
 
         assert!(key.starts_with(KEY_PREFIX), "{key}");
         assert_eq!(key.len(), KEY_PREFIX.len() + 64, "{key}");
-        for part in [PUBLISHER, "a_contributor", "secret"] {
+        for part in [PUBLISHER, "crawled.example", "a_contributor", "secret"] {
             assert!(!key.contains(part), "{key} should not hold {part}");
         }
     }
 
     #[test]
     fn parts_that_run_together_do_not_share_a_key() {
-        let one = held_key(
-            &Holder {
-                publisher: "ab",
-                id: "c",
-            },
-            "d",
-        );
-        let other = held_key(
-            &Holder {
-                publisher: "a",
-                id: "bc",
-            },
-            "d",
-        );
+        let key = |publisher, host, id| held_key(&Holder { publisher, id }, host, "d");
+
+        assert_ne!(key("ab", "h", "c"), key("a", "bh", "c"));
+        assert_ne!(key("p", "ab", "c"), key("p", "a", "bc"));
+        assert_ne!(key("p", "h", "c"), key("p", "other.example", "c"));
+    }
+
+    /// One deployment may serve several hosts, and each is answered for
+    /// itself.
+    #[tokio::test]
+    async fn an_answer_is_held_for_the_host_it_was_asked_for() {
+        let scripted = Arc::new(Scripted::new(vec![
+            Ok(rules("User-Agent: OneBot\nDisallow: /")),
+            Ok(rules("User-Agent: TwoBot\nDisallow: /")),
+        ]));
+        let services = services_with(Arc::default());
+        let other_host = http::Request::builder()
+            .uri("/robots.txt")
+            .header(http::header::HOST, "other.publisher.example")
+            .body(EdgeBody::empty())
+            .expect("should build the request");
+
+        ask(&scripted, &services, NOW)
+            .await
+            .expect("should ask the contributor");
+        let for_other_host = contribution_for(
+            &holder("a"),
+            &contributor(&scripted),
+            &ResolvedRequest::of(&other_host, &ClientInfo::default()),
+            &services,
+            NOW + 1,
+        )
+        .await
+        .expect("should ask for the other host");
 
-        assert_ne!(one, other);
+        assert_eq!(for_other_host, rules("User-Agent: TwoBot\nDisallow: /"));
     }
 
     #[tokio::test]
diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs
index 7f27ca1a4e..5e6531e14d 100644
--- a/crates/trusted-server-core/src/robots_txt/mod.rs
+++ b/crates/trusted-server-core/src/robots_txt/mod.rs
@@ -335,9 +335,14 @@ async fn answer(
         };
         selected.push(Some(contributor));
     }
-    // The answers are held and served to every crawler, so the contributors
-    // are handed the request with nothing one crawler put in the address.
-    let request = ResolvedRequest::of(req, services.client_info()).without_query();
+    // The answers are held and served to every crawler of the host, so the
+    // contributors are handed the request with nothing one crawler put in
+    // the address, for the host it was addressed to and never one a
+    // forwarded header names. That host is part of the key an answer is
+    // held under.
+    let request = ResolvedRequest::of(req, services.client_info())
+        .without_query()
+        .with_host(crate::http_util::addressed_host(req));
     Ok(respond(
         file_for(
             config,
@@ -356,7 +361,7 @@ async fn answer(
 type Selected = Option<(&'static str, Arc<dyn RobotsTxtContributor>)>;
 
 /// The file for `config`, from the contributions of `selected` in order,
-/// each asked for `request` and held for `publisher`.
+/// each asked for `request` and held for `publisher` and the request's host.
 async fn file_for(
     config: &RobotsTxtConfig,
     selected: &[Selected],
@@ -807,4 +812,97 @@ mod tests {
         assert!(message.contains("refuse_all"), "{message}");
         assert!(message.contains("allow_all"), "{message}");
     }
+
+    /// The body of a response, as text.
+    async fn text_of(response: Response<EdgeBody>) -> String {
+        let body = response
+            .into_body()
+            .into_bytes_bounded(4096)
+            .await
+            .expect("should read the body");
+        String::from_utf8(body.to_vec()).expect("the file is text")
+    }
+
+    /// A crawler's request for the file, addressed to `host`, carrying
+    /// `headers` as well.
+    fn robots_request_to(host: &str, headers: &[(&str, &str)]) -> Request<EdgeBody> {
+        let mut builder = Request::builder()
+            .uri(ROBOTS_TXT_PATH)
+            .header(header::HOST, host);
+        for &(name, value) in headers {
+            builder = builder.header(name, value);
+        }
+        builder
+            .body(EdgeBody::empty())
+            .expect("should build the request")
+    }
+
+    /// One deployment may serve several hosts, and each is answered with
+    /// what its contributors gave for that host, never with an answer held
+    /// for another.
+    #[tokio::test]
+    async fn each_host_is_answered_for_itself() {
+        let settings = settings_selecting_echo();
+        let registry = IntegrationRegistry::with_registrations(&settings, &[echo_builder()])
+            .expect("should build the registry");
+        let services = services_with(Arc::default());
+
+        for host in [PUBLISHER, "other.publisher.example"] {
+            let response = handle_robots_txt(
+                &settings,
+                &services,
+                &registry,
+                robots_request_to(host, &[]),
+            )
+            .await
+            .expect("should answer");
+            let file = text_of(response).await;
+
+            assert!(
+                file.contains(&format!("Disallow: /{host}/robots.txt?")),
+                "{host} should be answered for itself: {file}"
+            );
+        }
+    }
+
+    /// `Forwarded` and `X-Forwarded-Host` name whatever host a client likes,
+    /// so a contributor is asked for the host the request was addressed to,
+    /// and an answer made for a claimed host is never held for the real one.
+    #[tokio::test]
+    async fn a_forwarded_host_is_never_the_host_a_contributor_is_asked_for() {
+        let settings = settings_selecting_echo();
+        let registry = IntegrationRegistry::with_registrations(&settings, &[echo_builder()])
+            .expect("should build the registry");
+        let services = services_with(Arc::default());
+        let claiming = robots_request_to(
+            PUBLISHER,
+            &[
+                ("forwarded", "host=elsewhere.example"),
+                ("x-forwarded-host", "elsewhere.example"),
+            ],
+        );
+
+        let claimed = handle_robots_txt(&settings, &services, &registry, claiming)
+            .await
+            .expect("should answer");
+        let after = handle_robots_txt(
+            &settings,
+            &services,
+            &registry,
+            robots_request_to(PUBLISHER, &[]),
+        )
+        .await
+        .expect("should answer");
+
+        for file in [text_of(claimed).await, text_of(after).await] {
+            assert!(
+                !file.contains("elsewhere.example"),
+                "a claimed host should never reach the file: {file}"
+            );
+            assert!(
+                file.contains(&format!("Disallow: /{PUBLISHER}/robots.txt?")),
+                "the file should be made for the host addressed: {file}"
+            );
+        }
+    }
 }

From a752abe1344ae82a9600e0c52c4dae5d324a755c Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 16 Oct 2026 11:18:20 +0100
Subject: [PATCH 08/15] Answer a robots.txt with no rule with 503

A contributor that answered no group was an answer, so a file made only
of such answers was served as a 200 of one newline, cacheable for an hour
and held for the life of the stored answer. A crawler reads a file with no
rule as permission to crawl everything, which the section only means to
say with allow_all.

The assembled file is now read as a crawler reads it, and one in which no
group naming a crawler allows or disallows anything fails closed with the
same 503 as a contributor failure. The publisher's own rules in top_text or
bottom_text count, so a file whose rules come only from the publisher is
still served.
---
 .../src/robots_txt/contributor.rs             |   7 +-
 .../src/robots_txt/document.rs                |  37 +++++
 .../trusted-server-core/src/robots_txt/mod.rs | 145 +++++++++++++++++-
 3 files changed, 184 insertions(+), 5 deletions(-)

diff --git a/crates/trusted-server-core/src/robots_txt/contributor.rs b/crates/trusted-server-core/src/robots_txt/contributor.rs
index 39eacb1ca4..fb2b077e38 100644
--- a/crates/trusted-server-core/src/robots_txt/contributor.rs
+++ b/crates/trusted-server-core/src/robots_txt/contributor.rs
@@ -70,9 +70,10 @@ pub trait RobotsTxtContributor: Send + Sync + core::fmt::Debug {
     /// same for every request for that host, because it is served for all of
     /// them.
     ///
-    /// A contribution with no groups is an answer, and is served as one. An
-    /// error is a failure, and core serves this contributor's held copy
-    /// instead. With no held copy the whole file fails closed with `503`.
+    /// A contribution with no groups is an answer, and is held as one, but a
+    /// file in which nothing holds a rule fails closed with `503`. An error
+    /// is a failure, and core serves this contributor's held copy instead.
+    /// With no held copy the whole file fails closed with `503`.
     ///
     /// # Errors
     ///
diff --git a/crates/trusted-server-core/src/robots_txt/document.rs b/crates/trusted-server-core/src/robots_txt/document.rs
index 8973e61595..47a47c4e96 100644
--- a/crates/trusted-server-core/src/robots_txt/document.rs
+++ b/crates/trusted-server-core/src/robots_txt/document.rs
@@ -254,6 +254,20 @@ pub fn assemble(config: &RobotsTxtConfig, contributions: &[Contribution]) -> Str
     file
 }
 
+/// Whether `file` holds a rule, being an `Allow` or `Disallow` line in a group
+/// that names a crawler, read as a crawler reads the file. A file with none
+/// reads to a crawler as permission to crawl everything.
+#[must_use]
+pub(crate) fn holds_a_rule(file: &str) -> bool {
+    Contribution::parse(file).groups.iter().any(|group| {
+        group.user_agents.iter().any(|name| !name.is_empty())
+            && group
+                .records
+                .iter()
+                .any(|record| matches!(record, Record::Allow(_) | Record::Disallow(_)))
+    })
+}
+
 /// Every crawler the contributions refuse the whole site to, by the name on
 /// its `User-Agent` line, in order and without repeats. `*` is included, so
 /// the paths stay open to the crawlers no group names. A group with no name
@@ -525,6 +539,29 @@ mod tests {
         );
     }
 
+    /// Only an `Allow` or `Disallow` in a group that names a crawler is a
+    /// rule. Anything else leaves a crawler free to crawl everything.
+    #[test]
+    fn a_file_holds_a_rule_only_when_a_group_allows_or_disallows() {
+        for file in [
+            "\n",
+            "# nothing to add\n\nSitemap: https://publisher.example/sitemap.xml\n",
+            "Disallow: /\n",
+            "User-Agent: *\nCrawl-delay: 10\n",
+            "User-Agent: \nDisallow: /\n",
+        ] {
+            assert!(!holds_a_rule(file), "{file:?} holds no rule");
+        }
+        for file in [
+            "User-Agent: *\nAllow: /\n",
+            "User-Agent: ExampleBot\nDisallow:\n",
+            "# the publisher's own\nUser-agent: *\nDisallow: /private/\n",
+            refusal(&[]).as_str(),
+        ] {
+            assert!(holds_a_rule(file), "{file:?} holds a rule");
+        }
+    }
+
     #[test]
     fn refusing_with_no_allowances_refuses_everything() {
         assert_eq!(refusal(&[]), "User-agent: *\nDisallow: /\n");
diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs
index 5e6531e14d..fddb87442c 100644
--- a/crates/trusted-server-core/src/robots_txt/mod.rs
+++ b/crates/trusted-server-core/src/robots_txt/mod.rs
@@ -26,7 +26,8 @@
 //! 6. A contributor that fails with no held answer fails the whole file closed
 //!    with `503`, because leaving out a contributor's rules would allow by
 //!    omission whatever those rules refused. A `404` or an empty file reads to
-//!    a crawler as permission to crawl everything.
+//!    a crawler as permission to crawl everything, so a file with no rule
+//!    fails closed too, and permission is only said with `allow_all`.
 //!
 //! # Settings
 //!
@@ -362,6 +363,9 @@ type Selected = Option<(&'static str, Arc<dyn RobotsTxtContributor>)>;
 
 /// The file for `config`, from the contributions of `selected` in order,
 /// each asked for `request` and held for `publisher` and the request's host.
+///
+/// Fails, and the file is answered `503`, when a contributor fails with
+/// nothing of its held, and when the file would hold no rule.
 async fn file_for(
     config: &RobotsTxtConfig,
     selected: &[Selected],
@@ -380,7 +384,17 @@ async fn file_for(
             }
         });
     }
-    Ok(document::assemble(config, &contributions))
+    let file = document::assemble(config, &contributions);
+    // Permission to crawl everything is said with `allow_all`. A file with no
+    // rule says it too, by leaving everything out, so it fails closed.
+    if !document::holds_a_rule(&file) {
+        return Err(Report::new(configuration(
+            "[robots-txt] made a file with no rule, which a crawler reads as permission to \
+             crawl everything. Select `allow_all` to allow every crawler."
+                .to_owned(),
+        )));
+    }
+    Ok(file)
 }
 
 fn respond(file: Result<String, Report<TrustedServerError>>) -> Response<EdgeBody> {
@@ -905,4 +919,131 @@ mod tests {
             );
         }
     }
+
+    /// A contributor with nothing to add, which answers a comment and no
+    /// group.
+    #[derive(Debug)]
+    struct Silent;
+
+    #[async_trait::async_trait(?Send)]
+    impl RobotsTxtContributor for Silent {
+        fn refresh(&self) -> core::time::Duration {
+            core::time::Duration::from_secs(60)
+        }
+
+        fn fingerprint(&self) -> String {
+            "silent".to_owned()
+        }
+
+        async fn contribute(
+            &self,
+            _call: crate::module_context::ModuleCall<'_>,
+        ) -> Result<Contribution, Report<TrustedServerError>> {
+            Ok(Contribution {
+                preamble: vec![Record::Comment(" nothing to add".to_owned())],
+                groups: Vec::new(),
+            })
+        }
+    }
+
+    /// The id of the builder that registers the silent contributor.
+    const SILENT_BUILDER: &str = "robots-txt-silent";
+
+    /// The module name that builder is selected by.
+    const SILENT_BUILDER_MODULE: &str = "testing.robots-txt-silent";
+
+    fn register_silent(
+        _settings: &Settings,
+    ) -> Result<Option<IntegrationRegistration>, Report<TrustedServerError>> {
+        Ok(Some(
+            IntegrationRegistration::builder(SILENT_BUILDER)
+                .with_robots_txt_contributor("robots-txt.silent", Arc::new(Silent))
+                .build(),
+        ))
+    }
+
+    fn silent_builder() -> IntegrationBuilder {
+        IntegrationBuilder::new(SILENT_BUILDER, SILENT_BUILDER, register_silent, accept)
+            .with_module_name(SILENT_BUILDER_MODULE)
+    }
+
+    /// The test settings with `robots_txt` as the section and the test
+    /// builder named `module` selected.
+    fn settings_selecting(robots_txt: serde_json::Value, module: &str) -> Settings {
+        let mut settings = settings();
+        settings.robots_txt = Some(parsed(robots_txt));
+        settings
+            .insert_module_config("testing", module, &serde_json::json!({}))
+            .expect("should select the test builder");
+        settings
+    }
+
+    /// A file with no rule reads to a crawler as permission to crawl
+    /// everything, which is only ever said with `allow_all`, so the file
+    /// fails closed when its sources give no rule.
+    #[tokio::test]
+    async fn a_file_with_no_rule_is_service_unavailable() {
+        let settings = settings_selecting(
+            serde_json::json!({
+                "modules": ["silent"],
+                "sitemap": "https://www.publisher.example/sitemap.xml",
+            }),
+            SILENT_BUILDER_MODULE,
+        );
+        let registry = IntegrationRegistry::with_registrations(&settings, &[silent_builder()])
+            .expect("should build the registry");
+
+        let response = handle_robots_txt(
+            &settings,
+            &services_with(Arc::default()),
+            &registry,
+            robots_request(),
+        )
+        .await
+        .expect("should answer");
+
+        assert_eq!(
+            response.status(),
+            StatusCode::SERVICE_UNAVAILABLE,
+            "a file with no rule should never be served"
+        );
+        assert_eq!(
+            response.headers().get(header::RETRY_AFTER),
+            Some(&HeaderValue::from_static("600"))
+        );
+        assert_eq!(
+            response.headers().get(header::CACHE_CONTROL),
+            Some(&HeaderValue::from_static("no-store"))
+        );
+    }
+
+    /// The publisher's own text says what the publisher wants, so a file
+    /// whose only rules are the publisher's is served.
+    #[tokio::test]
+    async fn the_publishers_own_rules_are_served_when_no_contributor_adds_any() {
+        let settings = settings_selecting(
+            serde_json::json!({
+                "modules": ["silent"],
+                "top_text": "User-agent: *\nDisallow: /private/",
+            }),
+            SILENT_BUILDER_MODULE,
+        );
+        let registry = IntegrationRegistry::with_registrations(&settings, &[silent_builder()])
+            .expect("should build the registry");
+
+        let response = handle_robots_txt(
+            &settings,
+            &services_with(Arc::default()),
+            &registry,
+            robots_request(),
+        )
+        .await
+        .expect("should answer");
+
+        assert_eq!(response.status(), StatusCode::OK);
+        assert_eq!(
+            text_of(response).await,
+            "User-agent: *\nDisallow: /private/\n\n# nothing to add\n"
+        );
+    }
 }

From 5fbe25b2cbfd1f6c6b11e89476d6b171a9f83f1b Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 16 Oct 2026 11:21:49 +0100
Subject: [PATCH 09/15] Give a robots.txt contributor nothing of the crawler's
 connection

The context a contributor was asked through carried no client data, but
the services it was handed did: client_info gave the crawler's address,
and where the adapter supplies them its TLS and HTTP/2 details and the edge
server, and host_signals gave the host's client signals. A contributor
naming the services could shape the answer held and served to every
crawler from the one that asked.

The contributor is now handed a copy of the services with default client
information and no host signals. The stores, backends and HTTP client are
the request's as before.
---
 .../trusted-server-core/src/platform/types.rs |  14 +++
 .../src/robots_txt/contributor.rs             |  10 +-
 .../src/robots_txt/holding.rs                 |   5 +-
 .../trusted-server-core/src/robots_txt/mod.rs | 109 ++++++++++++++++++
 4 files changed, 132 insertions(+), 6 deletions(-)

diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs
index 837873359c..95c3ed7c0c 100644
--- a/crates/trusted-server-core/src/platform/types.rs
+++ b/crates/trusted-server-core/src/platform/types.rs
@@ -405,6 +405,20 @@ impl RuntimeServices {
         }
     }
 
+    /// Returns a clone of this instance with nothing of the connection it was
+    /// built for, being default client metadata and no host signals.
+    ///
+    /// For a module whose answer is held and served to everyone who asks, so
+    /// that nothing of one connection can reach what the others are given.
+    #[must_use]
+    pub(crate) fn without_connection(self) -> Self {
+        Self {
+            client_info: ClientInfo::default(),
+            host_signals: None,
+            ..self
+        }
+    }
+
     /// Returns a clone of this instance with the Edge Cookie module replaced.
     ///
     /// Adapters use this to apply the module-supplied module selected by
diff --git a/crates/trusted-server-core/src/robots_txt/contributor.rs b/crates/trusted-server-core/src/robots_txt/contributor.rs
index fb2b077e38..773ec0560f 100644
--- a/crates/trusted-server-core/src/robots_txt/contributor.rs
+++ b/crates/trusted-server-core/src/robots_txt/contributor.rs
@@ -64,11 +64,11 @@ pub trait RobotsTxtContributor: Send + Sync + core::fmt::Debug {
     /// addressed to, and serves it to every crawler of that host. So the call
     /// carries the request the answer was asked for, being its method, path,
     /// host and scheme without its query, with the host never taken from a
-    /// forwarded header, and the request's services, and nothing about who
-    /// asked. An implementation hands its own function to
-    /// [`ModuleCall::inject`], naming what it needs. Its answer must be the
-    /// same for every request for that host, because it is served for all of
-    /// them.
+    /// forwarded header, and the request's services with no client
+    /// information and no host signals, and nothing about who asked. An
+    /// implementation hands its own function to [`ModuleCall::inject`],
+    /// naming what it needs. Its answer must be the same for every request
+    /// for that host, because it is served for all of them.
     ///
     /// A contribution with no groups is an answer, and is held as one, but a
     /// file in which nothing holds a rule fails closed with `503`. An error
diff --git a/crates/trusted-server-core/src/robots_txt/holding.rs b/crates/trusted-server-core/src/robots_txt/holding.rs
index 0cf5d1a679..66e3727b08 100644
--- a/crates/trusted-server-core/src/robots_txt/holding.rs
+++ b/crates/trusted-server-core/src/robots_txt/holding.rs
@@ -82,7 +82,10 @@ pub(crate) async fn contribution_for(
     {
         return Ok(held.contribution.clone());
     }
-    let context = ModuleContext::new(request.view()).with_services(services);
+    // The answer is served to every crawler, so the contributor's services
+    // carry nothing of the connection that asked.
+    let services_handed = services.clone().without_connection();
+    let context = ModuleContext::new(request.view()).with_services(&services_handed);
     let asked = contributor
         .contribute(context.call(holder.id, contributor.required_permissions()))
         .await;
diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs
index fddb87442c..f3f8a21ab4 100644
--- a/crates/trusted-server-core/src/robots_txt/mod.rs
+++ b/crates/trusted-server-core/src/robots_txt/mod.rs
@@ -1046,4 +1046,113 @@ mod tests {
             "User-agent: *\nDisallow: /private/\n\n# nothing to add\n"
         );
     }
+
+    /// A contributor whose rule says what it could read of the connection
+    /// that asked.
+    #[derive(Debug)]
+    struct ReadsTheConnection;
+
+    impl ReadsTheConnection {
+        fn rules(&self, services: &RuntimeServices) -> Contribution {
+            let client = services.client_info();
+            Contribution::parse(&format!(
+                "User-Agent: ConnectionBot\nDisallow: /ip-{}/tls-{}/ja4-{}/edge-{}/signals-{}",
+                client.client_ip.is_some(),
+                client.tls_protocol.is_some(),
+                client.tls_ja4.is_some(),
+                client.server_hostname.is_some(),
+                services.host_signals().is_some(),
+            ))
+        }
+    }
+
+    #[async_trait::async_trait(?Send)]
+    impl RobotsTxtContributor for ReadsTheConnection {
+        fn refresh(&self) -> core::time::Duration {
+            core::time::Duration::from_secs(60)
+        }
+
+        fn fingerprint(&self) -> String {
+            "connection".to_owned()
+        }
+
+        async fn contribute(
+            &self,
+            call: crate::module_context::ModuleCall<'_>,
+        ) -> Result<Contribution, Report<TrustedServerError>> {
+            Ok(call.inject(self, Self::rules)?)
+        }
+    }
+
+    /// The signals a host computes from a crawler's connection.
+    #[derive(Debug)]
+    struct CrawlerSignals;
+
+    impl crate::evidence::HostSignals for CrawlerSignals {
+        fn ja4(&self) -> Option<&str> {
+            Some("t13d1516h2_8daaf6152771_02713d6af862")
+        }
+
+        fn h2(&self) -> Option<&str> {
+            Some("1:65536;2:0;4:6291456")
+        }
+    }
+
+    /// The id of the builder that registers the connection reading
+    /// contributor.
+    const CONNECTION_BUILDER: &str = "robots-txt-connection";
+
+    /// The module name that builder is selected by.
+    const CONNECTION_BUILDER_MODULE: &str = "testing.robots-txt-connection";
+
+    fn register_connection(
+        _settings: &Settings,
+    ) -> Result<Option<IntegrationRegistration>, Report<TrustedServerError>> {
+        Ok(Some(
+            IntegrationRegistration::builder(CONNECTION_BUILDER)
+                .with_robots_txt_contributor("robots-txt.connection", Arc::new(ReadsTheConnection))
+                .build(),
+        ))
+    }
+
+    /// The answer is served to every crawler, so the services a contributor
+    /// is handed carry nothing of the connection that asked: no address, no
+    /// TLS details, no edge server and no host signals.
+    #[tokio::test]
+    async fn a_contributor_is_handed_nothing_of_the_connection_that_asked() {
+        let settings = settings_selecting(
+            serde_json::json!({ "modules": ["connection"] }),
+            CONNECTION_BUILDER_MODULE,
+        );
+        let builder = IntegrationBuilder::new(
+            CONNECTION_BUILDER,
+            CONNECTION_BUILDER,
+            register_connection,
+            accept,
+        )
+        .with_module_name(CONNECTION_BUILDER_MODULE);
+        let registry = IntegrationRegistry::with_registrations(&settings, &[builder])
+            .expect("should build the registry");
+        let connection = crate::platform::ClientInfo {
+            client_ip: Some("203.0.113.7".parse().expect("should parse the address")),
+            tls_protocol: Some("TLSv1.3".to_owned()),
+            tls_ja4: Some("t13d1516h2_8daaf6152771_02713d6af862".to_owned()),
+            server_hostname: Some("cache-edge-1".to_owned()),
+            ..crate::platform::ClientInfo::default()
+        };
+        let services = RuntimeServices {
+            host_signals: Some(Arc::new(CrawlerSignals)),
+            ..services_with(Arc::default()).with_client_info(connection)
+        };
+
+        let response = handle_robots_txt(&settings, &services, &registry, robots_request())
+            .await
+            .expect("should answer");
+        let file = text_of(response).await;
+
+        assert!(
+            file.contains("Disallow: /ip-false/tls-false/ja4-false/edge-false/signals-false"),
+            "the contributor should read nothing of the connection: {file}"
+        );
+    }
 }

From 4805bfca8057a7f6ec72673557b01b361c5ce01b Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 16 Oct 2026 11:25:18 +0100
Subject: [PATCH 10/15] Answer robots.txt on Fastly before the Edge Cookie
 steps and the filters

The route ran as every named route does on this adapter: the request
preparers, then the Edge Cookie request state, which reads the crawler's
cookies and consent signals and asks the geo module for its location, then
the request filters, and the Edge Cookie finalize step on the way out. A
filter that answers, such as a bot challenge, stood in for the file, and a
crawler reads a 4xx for robots.txt as permission to crawl everything.

The file is now answered once the modules have prepared the request, as the
configuration, permissions and data pages are answered before the Edge
Cookie steps, and no Edge Cookie state rides on the response.
---
 .../trusted-server-adapter-fastly/src/app.rs  | 13 ++-
 .../src/app/seam_probe_tests.rs               | 99 ++++++++++++++++++-
 2 files changed, 108 insertions(+), 4 deletions(-)

diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs
index e649b19d3f..c4cd4e0a16 100644
--- a/crates/trusted-server-adapter-fastly/src/app.rs
+++ b/crates/trusted-server-adapter-fastly/src/app.rs
@@ -795,6 +795,17 @@ async fn execute_named(
         return Ok(http_error(&report));
     }
 
+    // A crawler asking for robots.txt is answered once the modules have
+    // prepared the request, before the Edge Cookie steps and the request
+    // filters, so nothing is read from its cookies or written back, and no
+    // filter's answer stands in for the file.
+    if matches!(handler, NamedRouteHandler::RobotsTxt) {
+        let response = handle_robots_txt(&state.settings, &services, &state.registry, req)
+            .await
+            .unwrap_or_else(|error| http_error(&error));
+        return Ok(response);
+    }
+
     let mut ec = build_ec_request_state(&state.settings, &services, &req).await;
     // EcContext creation errors short-circuit before filters, mirroring legacy:
     // the legacy path returns its error response before running filter_request.
@@ -939,7 +950,7 @@ async fn run_named_route(
             handle_first_party_proxy_rebuild(&state.settings, services, req).await
         }
         NamedRouteHandler::RobotsTxt => {
-            handle_robots_txt(&state.settings, services, &state.registry, req).await
+            unreachable!("robots.txt is answered before EC setup")
         }
     }
 }
diff --git a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs
index 39ad2d99f7..4a6e57a468 100644
--- a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs
+++ b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs
@@ -16,7 +16,10 @@ use error_stack::Report;
 use futures::executor::block_on;
 use trusted_server_core::error::TrustedServerError;
 use trusted_server_core::evidence::OwnedRequestInfo;
-use trusted_server_core::integrations::{IntegrationBuilder, IntegrationRegistration};
+use trusted_server_core::integrations::{
+    IntegrationBuilder, IntegrationRegistration, IntegrationRequestFilter, RequestFilterDecision,
+    RequestFilterEffects, RequestFilterInput,
+};
 use trusted_server_core::module_context::{ModuleContext, ModuleRequest};
 use trusted_server_core::settings::Settings;
 use trusted_server_core::tsjs::tsjs_script_src;
@@ -24,8 +27,8 @@ use trusted_server_core::tsjs_bundle::{JsModulePart, compile_time_parts};
 use trusted_server_integration_seam_probe as seam_probe;
 
 use super::{
-    AppState, TrustedServerApp, build_finalize_services, build_state_from_settings,
-    build_state_with_registrations, register_integrations,
+    AppState, EcFinalizeState, TrustedServerApp, build_finalize_services,
+    build_state_from_settings, build_state_with_registrations, register_integrations,
 };
 
 /// Largest response body these tests read, ample for the served script.
@@ -347,6 +350,96 @@ fn robots_txt_is_this_servers_only_when_the_settings_carry_the_section() {
     }
 }
 
+/// The id of a module whose request filter answers every request with a
+/// challenge, as bot protection does.
+const CHALLENGE_ID: &str = "robots-txt-challenge";
+
+/// The module name that module is selected by.
+const CHALLENGE_MODULE: &str = "testing.robots-txt-challenge";
+
+/// Answers every request with a `403` challenge.
+struct Challenging;
+
+#[async_trait::async_trait(?Send)]
+impl IntegrationRequestFilter for Challenging {
+    fn integration_id(&self) -> &'static str {
+        CHALLENGE_ID
+    }
+
+    async fn filter_request(
+        &self,
+        _input: RequestFilterInput<'_>,
+    ) -> Result<RequestFilterDecision, Report<TrustedServerError>> {
+        let response = edgezero_core::http::response_builder()
+            .status(403)
+            .body(Body::from("a challenge"))
+            .expect("should build the challenge");
+        Ok(RequestFilterDecision::Respond {
+            response: Box::new(response),
+            effects: RequestFilterEffects::default(),
+        })
+    }
+}
+
+fn register_challenge(
+    _settings: &Settings,
+) -> Result<Option<IntegrationRegistration>, Report<TrustedServerError>> {
+    Ok(Some(
+        IntegrationRegistration::builder(CHALLENGE_ID)
+            .with_request_filter(Arc::new(Challenging))
+            .build(),
+    ))
+}
+
+fn accept_challenge(_settings: &Settings) -> Result<bool, Report<TrustedServerError>> {
+    Ok(true)
+}
+
+/// A crawler asking for robots.txt is answered before the Edge Cookie steps
+/// and the request filters, so a filter that answers every other request with
+/// a challenge never stands in for the file, and no Edge Cookie state rides
+/// on the answer.
+#[test]
+fn robots_txt_is_answered_before_the_edge_cookie_steps_and_the_filters() {
+    let settings = settings_with(&format!(
+        r#"
+        [robots-txt]
+        modules = "refuse_all"
+        {HMAC_BLOCK}
+
+        [testing]
+        modules = ["robots-txt-challenge"]
+        "#
+    ));
+    let challenge = IntegrationBuilder::new(
+        CHALLENGE_ID,
+        CHALLENGE_ID,
+        register_challenge,
+        accept_challenge,
+    )
+    .with_module_name(CHALLENGE_MODULE);
+    let router = router_with(settings, &[challenge]);
+
+    let other = get(&router, "/.well-known/trusted-server.json", None);
+    assert_eq!(
+        other.status().as_u16(),
+        403,
+        "the filter should answer another named route"
+    );
+
+    let file = get(&router, "/robots.txt", None);
+    assert_eq!(
+        file.status().as_u16(),
+        200,
+        "the file should be served, not the filter's challenge"
+    );
+    assert!(
+        file.extensions().get::<EcFinalizeState>().is_none(),
+        "no Edge Cookie state should ride on the file"
+    );
+    assert_eq!(body_text(file), "User-agent: *\nDisallow: /\n");
+}
+
 /// A request prepares exactly once whether a named route or the fallback
 /// serves it.
 #[test]

From e8a2bf576cac9dd56f8c4d14fd18f7835ef18e0f Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 16 Oct 2026 11:33:06 +0100
Subject: [PATCH 11/15] Test robots.txt through every adapter: HEAD, a held
 answer and the 503

The adapter tests covered the route's registration, and only Fastly and
Axum asked a module for rules. These tests drive the file through each
router.

- Fastly and Axum answer a HEAD with the headers of the GET and no body.
- On Fastly and Axum, which hold nothing, a module that answered once and
  then fails makes the next request a 503 with Retry-After: 600 and
  Cache-Control: no-store.
- On Cloudflare and Spin a module's rules are served, and with the store
  the runtime hands the application its answer is served again when it
  next fails. With no store the file is the 503.
- On Axum, which leaves forwarded host headers in place, a claimed host
  never reaches a module.

The core test of the 503 now checks that Retry-After is 600.
---
 .../tests/seam_probe.rs                       | 194 ++++++++++++++-
 .../tests/routes.rs                           | 235 ++++++++++++++++++
 .../src/app/seam_probe_tests.rs               | 140 ++++++++++-
 .../tests/routes.rs                           | 235 ++++++++++++++++++
 .../trusted-server-core/src/robots_txt/mod.rs |   6 +-
 5 files changed, 807 insertions(+), 3 deletions(-)

diff --git a/crates/trusted-server-adapter-axum/tests/seam_probe.rs b/crates/trusted-server-adapter-axum/tests/seam_probe.rs
index bb815ed33f..494d70add8 100644
--- a/crates/trusted-server-adapter-axum/tests/seam_probe.rs
+++ b/crates/trusted-server-adapter-axum/tests/seam_probe.rs
@@ -23,12 +23,13 @@ use trusted_server_core::evidence::OwnedRequestInfo;
 use trusted_server_core::integrations::{
     IntegrationBuilder, IntegrationRegistration, IntegrationRegistry,
 };
-use trusted_server_core::module_context::{ModuleContext, ModuleRequest};
+use trusted_server_core::module_context::{ModuleCall, ModuleContext, ModuleRequest};
 use trusted_server_core::platform::{
     BackendNamingPolicy, ClientInfo, DisabledGeo, PlatformBackend, PlatformBackendSpec,
     PlatformConfigStore, PlatformError, PlatformSecretStore, RuntimeServices, StoreId, StoreName,
     UnavailableHttpClient, UnavailableKvStore,
 };
+use trusted_server_core::robots_txt::{Contribution, RobotsTxtContributor};
 use trusted_server_core::settings::Settings;
 use trusted_server_core::tsjs::tsjs_script_src;
 use trusted_server_core::tsjs_bundle::{JsModulePart, compile_time_parts};
@@ -411,6 +412,197 @@ fn robots_txt_is_this_servers_only_when_the_settings_carry_the_section() {
     }
 }
 
+/// Sends one request with `method` and `headers` and returns the response.
+async fn send(
+    service: &mut EdgeZeroAxumService,
+    method: &str,
+    uri: &str,
+    headers: &[(&str, &str)],
+) -> axum::response::Response {
+    let mut builder = Request::builder().method(method).uri(uri);
+    for &(name, value) in headers {
+        builder = builder.header(name, value);
+    }
+    let request = builder
+        .body(AxumBody::empty())
+        .expect("should build request");
+
+    service
+        .ready()
+        .await
+        .expect("should be ready")
+        .call(request)
+        .await
+        .expect("should respond")
+}
+
+/// The value of the header `name`, as text.
+fn header_text(response: &axum::response::Response, name: &str) -> Option<String> {
+    response
+        .headers()
+        .get(name)
+        .and_then(|value| value.to_str().ok())
+        .map(str::to_owned)
+}
+
+/// A `HEAD` for the file is answered here with the headers a `GET` is given
+/// and no body.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_head_of_robots_txt_is_answered_with_the_headers_and_no_body() {
+    let settings = settings_with(&format!(
+        r#"
+            [robots-txt]
+            modules = "refuse_all"
+            {PROBE_BLOCK}
+        "#
+    ));
+    let mut service = service_with(settings, &[seam_probe::builder()]);
+
+    let response = send(&mut service, "HEAD", "/robots.txt", &[]).await;
+
+    assert_eq!(response.status().as_u16(), 200);
+    assert_eq!(
+        header_text(&response, "content-type").as_deref(),
+        Some("text/plain; charset=utf-8")
+    );
+    assert_eq!(
+        header_text(&response, "cache-control").as_deref(),
+        Some("public, max-age=3600")
+    );
+    assert!(
+        body_text(response).await.is_empty(),
+        "a HEAD carries no body"
+    );
+}
+
+/// The id of the module that answers once.
+const ANSWERS_ONCE_ID: &str = "robots-txt-answers-once";
+
+/// A robots.txt contributor that answers the first time it is asked and fails
+/// every time after, whose answer is due for a refresh at once.
+#[derive(Debug, Default)]
+struct AnswersOnce {
+    asked: std::sync::atomic::AtomicUsize,
+}
+
+#[async_trait::async_trait(?Send)]
+impl RobotsTxtContributor for AnswersOnce {
+    fn refresh(&self) -> std::time::Duration {
+        std::time::Duration::ZERO
+    }
+
+    fn fingerprint(&self) -> String {
+        "answers-once".to_owned()
+    }
+
+    async fn contribute(
+        &self,
+        _call: ModuleCall<'_>,
+    ) -> Result<Contribution, Report<TrustedServerError>> {
+        if self.asked.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
+            Ok(Contribution::parse("User-Agent: OnceBot\nDisallow: /"))
+        } else {
+            Err(Report::new(TrustedServerError::Proxy {
+                message: "the rules service is down".to_owned(),
+            }))
+        }
+    }
+}
+
+fn register_answers_once(
+    _settings: &Settings,
+) -> Result<Option<IntegrationRegistration>, Report<TrustedServerError>> {
+    Ok(Some(
+        IntegrationRegistration::builder(ANSWERS_ONCE_ID)
+            .with_robots_txt_contributor(
+                "robots-txt.answers-once",
+                Arc::new(AnswersOnce::default()),
+            )
+            .build(),
+    ))
+}
+
+fn accept_answers_once(_settings: &Settings) -> Result<bool, Report<TrustedServerError>> {
+    Ok(true)
+}
+
+/// Settings whose file is made by the module that answers once.
+const ANSWERS_ONCE_BLOCK: &str = r#"
+            [robots-txt]
+            modules = ["answers-once"]
+
+            [testing]
+            modules = ["robots-txt-answers-once"]
+"#;
+
+/// This adapter holds no answer, so every request for the file asks each
+/// module, and a module that fails with nothing held makes the file a `503`
+/// that tells a crawler to come back in ten minutes and is not stored.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn with_nothing_held_a_module_that_fails_makes_the_file_unavailable() {
+    let builder = IntegrationBuilder::new(
+        ANSWERS_ONCE_ID,
+        ANSWERS_ONCE_ID,
+        register_answers_once,
+        accept_answers_once,
+    )
+    .with_module_name("testing.robots-txt-answers-once");
+    let mut service = service_with(settings_with(ANSWERS_ONCE_BLOCK), &[builder]);
+
+    let first = get(&mut service, "/robots.txt").await;
+    assert_eq!(first.status().as_u16(), 200);
+    assert_eq!(body_text(first).await, "User-Agent: OnceBot\nDisallow: /\n");
+
+    let second = get(&mut service, "/robots.txt").await;
+    assert_eq!(
+        second.status().as_u16(),
+        503,
+        "nothing is held here, so the module is asked again and its failure fails the file"
+    );
+    assert_eq!(header_text(&second, "retry-after").as_deref(), Some("600"));
+    assert_eq!(
+        header_text(&second, "cache-control").as_deref(),
+        Some("no-store")
+    );
+}
+
+/// This adapter leaves `Forwarded` and `X-Forwarded-Host` in place, and the
+/// probe's rule names the host it was asked for, so the file shows that a
+/// claimed host never reaches a module.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_forwarded_host_never_reaches_a_robots_txt_module() {
+    let settings = settings_with(&format!(
+        r#"
+            [robots-txt]
+            modules = ["testing.seam-probe"]
+            {PROBE_BLOCK}
+        "#
+    ));
+    let mut service = service_with(settings, &[seam_probe::builder()]);
+
+    let response = send(
+        &mut service,
+        "GET",
+        "/robots.txt",
+        &[
+            ("host", "test-publisher.example.com"),
+            ("x-forwarded-host", "elsewhere.example"),
+            ("forwarded", "host=elsewhere.example"),
+        ],
+    )
+    .await;
+    let file = body_text(response).await;
+
+    assert!(
+        file.contains("Disallow: /test-publisher.example.com/robots.txt?"),
+        "the probe should be asked for the host addressed: {file}"
+    );
+    assert!(
+        !file.contains("elsewhere.example"),
+        "a claimed host should never reach the file: {file}"
+    );
+}
+
 /// A request prepares exactly once whether it is served by a named route or by
 /// the fallback, so the invariant holds across the whole route table rather
 /// than only on the path the probe's own proxy route sits on.
diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs
index fae32e6683..8594dc1a07 100644
--- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs
+++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs
@@ -728,3 +728,238 @@ async fn a_response_carries_no_blanket_rule_without_the_section() {
 
     assert_eq!(robots_tag(&response), None);
 }
+
+/// A module's rules served through this adapter's router, held in the
+/// key-value store the runtime hands the application.
+mod robots_txt_from_a_module {
+    use std::collections::HashMap;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::{Arc, Mutex};
+    use std::time::Duration;
+
+    use bytes::Bytes;
+    use edgezero_core::http::Request;
+    use edgezero_core::key_value_store::{KvError, KvHandle, KvPage, KvStore};
+    use edgezero_core::router::RouterService;
+    use edgezero_core::store_registry::KvRegistry;
+    use error_stack::Report;
+    use trusted_server_adapter_cloudflare::app::TrustedServerApp;
+    use trusted_server_core::error::TrustedServerError;
+    use trusted_server_core::integrations::{IntegrationBuilder, IntegrationRegistration};
+    use trusted_server_core::module_context::ModuleCall;
+    use trusted_server_core::robots_txt::{Contribution, RobotsTxtContributor};
+    use trusted_server_core::settings::Settings;
+
+    use super::robots_request;
+
+    /// The id of the module that answers once.
+    const ANSWERS_ONCE_ID: &str = "robots-txt-answers-once";
+
+    /// A robots.txt contributor that answers the first time it is asked and
+    /// fails every time after, whose answer is due for a refresh at once.
+    #[derive(Debug, Default)]
+    struct AnswersOnce {
+        asked: AtomicUsize,
+    }
+
+    #[async_trait::async_trait(?Send)]
+    impl RobotsTxtContributor for AnswersOnce {
+        fn refresh(&self) -> Duration {
+            Duration::ZERO
+        }
+
+        fn fingerprint(&self) -> String {
+            "answers-once".to_owned()
+        }
+
+        async fn contribute(
+            &self,
+            _call: ModuleCall<'_>,
+        ) -> Result<Contribution, Report<TrustedServerError>> {
+            if self.asked.fetch_add(1, Ordering::SeqCst) == 0 {
+                Ok(Contribution::parse("User-Agent: OnceBot\nDisallow: /"))
+            } else {
+                Err(Report::new(TrustedServerError::Proxy {
+                    message: "the rules service is down".to_owned(),
+                }))
+            }
+        }
+    }
+
+    fn register_answers_once(
+        _settings: &Settings,
+    ) -> Result<Option<IntegrationRegistration>, Report<TrustedServerError>> {
+        Ok(Some(
+            IntegrationRegistration::builder(ANSWERS_ONCE_ID)
+                .with_robots_txt_contributor(
+                    "robots-txt.answers-once",
+                    Arc::new(AnswersOnce::default()),
+                )
+                .build(),
+        ))
+    }
+
+    fn accept_answers_once(_settings: &Settings) -> Result<bool, Report<TrustedServerError>> {
+        Ok(true)
+    }
+
+    /// The route test settings with a file made by the module that answers
+    /// once.
+    fn router_with_a_module_that_answers_once() -> RouterService {
+        let settings = Settings::from_toml(
+            r#"
+                [publisher]
+                domain = "test-publisher.example.com"
+                cookie_domain = ".test-publisher.example.com"
+                origin_url = "https://origin.test-publisher.example.com"
+                proxy_secret = "route-test-proxy-secret"
+
+                [ec]
+                module = "hmac"
+
+                [ec.hmac]
+                passphrase = "test-secret-key-32-bytes-minimum"
+
+                [geo]
+                assume_single_jurisdiction = true
+
+                [robots-txt]
+                modules = ["answers-once"]
+
+                [testing]
+                modules = ["robots-txt-answers-once"]
+            "#,
+        )
+        .expect("should parse route test settings");
+        let builder = IntegrationBuilder::new(
+            ANSWERS_ONCE_ID,
+            ANSWERS_ONCE_ID,
+            register_answers_once,
+            accept_answers_once,
+        )
+        .with_module_name("testing.robots-txt-answers-once");
+
+        TrustedServerApp::routes_with_registrations(settings, &[builder])
+            .expect("should build router from test settings")
+    }
+
+    /// A key-value store kept in memory for one test, so what one request
+    /// writes is what the next one reads.
+    #[derive(Default)]
+    struct MemoryStore {
+        entries: Mutex<HashMap<String, Bytes>>,
+    }
+
+    #[async_trait::async_trait(?Send)]
+    impl KvStore for MemoryStore {
+        async fn delete(&self, key: &str) -> Result<(), KvError> {
+            self.entries
+                .lock()
+                .expect("should lock the store")
+                .remove(key);
+            Ok(())
+        }
+
+        async fn get_bytes(&self, key: &str) -> Result<Option<Bytes>, KvError> {
+            Ok(self
+                .entries
+                .lock()
+                .expect("should lock the store")
+                .get(key)
+                .cloned())
+        }
+
+        async fn list_keys_page(
+            &self,
+            _prefix: &str,
+            _cursor: Option<&str>,
+            _limit: usize,
+        ) -> Result<KvPage, KvError> {
+            Ok(KvPage::default())
+        }
+
+        async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> {
+            self.entries
+                .lock()
+                .expect("should lock the store")
+                .insert(key.to_owned(), value);
+            Ok(())
+        }
+
+        async fn put_bytes_with_ttl(
+            &self,
+            key: &str,
+            value: Bytes,
+            _ttl: Duration,
+        ) -> Result<(), KvError> {
+            self.put_bytes(key, value).await
+        }
+    }
+
+    /// A crawler's `GET` for the file, carrying `store` as the key-value store
+    /// the runtime hands the application, when there is one.
+    fn robots_get(store: Option<&Arc<MemoryStore>>) -> Request {
+        let mut request = robots_request("GET", "/robots.txt");
+        if let Some(store) = store {
+            let handle = KvHandle::new(Arc::clone(store) as Arc<dyn KvStore>);
+            request.extensions_mut().insert(KvRegistry::single_id(
+                "trusted_server_kv".to_owned(),
+                handle,
+            ));
+        }
+        request
+    }
+
+    /// With the store the runtime hands the application, a module's answer
+    /// is held, so when the module next fails the file is made from the
+    /// answer it gave before.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn a_held_answer_is_served_when_its_module_fails() {
+        let router = router_with_a_module_that_answers_once();
+        let store = Arc::new(MemoryStore::default());
+
+        for asked in ["answers", "fails"] {
+            let response = router
+                .oneshot(robots_get(Some(&store)))
+                .await
+                .expect("should route request");
+
+            assert_eq!(response.status().as_u16(), 200, "the module {asked}");
+            let body = response.into_body().into_bytes().unwrap_or_default();
+            assert_eq!(
+                &body[..],
+                b"User-Agent: OnceBot\nDisallow: /\n",
+                "the module {asked}"
+            );
+        }
+    }
+
+    /// With no store from the runtime nothing is held, so a module that fails
+    /// makes the file a `503` that tells a crawler to come back in ten
+    /// minutes and is not stored.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn with_nothing_held_a_module_that_fails_makes_the_file_unavailable() {
+        let router = router_with_a_module_that_answers_once();
+
+        let first = router
+            .oneshot(robots_get(None))
+            .await
+            .expect("should route request");
+        assert_eq!(first.status().as_u16(), 200);
+
+        let second = router
+            .oneshot(robots_get(None))
+            .await
+            .expect("should route request");
+        assert_eq!(second.status().as_u16(), 503);
+        let header = |name: &str| {
+            second
+                .headers()
+                .get(name)
+                .and_then(|value| value.to_str().ok())
+                .map(str::to_owned)
+        };
+        assert_eq!(header("retry-after").as_deref(), Some("600"));
+        assert_eq!(header("cache-control").as_deref(), Some("no-store"));
+    }
+}
diff --git a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs
index 4a6e57a468..a21b59a635 100644
--- a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs
+++ b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs
@@ -20,7 +20,8 @@ use trusted_server_core::integrations::{
     IntegrationBuilder, IntegrationRegistration, IntegrationRequestFilter, RequestFilterDecision,
     RequestFilterEffects, RequestFilterInput,
 };
-use trusted_server_core::module_context::{ModuleContext, ModuleRequest};
+use trusted_server_core::module_context::{ModuleCall, ModuleContext, ModuleRequest};
+use trusted_server_core::robots_txt::{Contribution, RobotsTxtContributor};
 use trusted_server_core::settings::Settings;
 use trusted_server_core::tsjs::tsjs_script_src;
 use trusted_server_core::tsjs_bundle::{JsModulePart, compile_time_parts};
@@ -440,6 +441,143 @@ fn robots_txt_is_answered_before_the_edge_cookie_steps_and_the_filters() {
     assert_eq!(body_text(file), "User-agent: *\nDisallow: /\n");
 }
 
+/// Sends one request with `method` and returns the response.
+fn send(router: &RouterService, method: Method, path: &str) -> Response {
+    let request = request_builder()
+        .method(method)
+        .uri(format!("https://test-publisher.com{path}"))
+        .body(Body::empty())
+        .expect("should build request");
+    block_on(router.oneshot(request)).expect("should route request")
+}
+
+/// The value of the header `name`, as text.
+fn header_text<'r>(response: &'r Response, name: &str) -> Option<&'r str> {
+    response
+        .headers()
+        .get(name)
+        .and_then(|value| value.to_str().ok())
+}
+
+/// A `HEAD` for the file is answered here with the headers a `GET` is given
+/// and no body.
+#[test]
+fn a_head_of_robots_txt_is_answered_with_the_headers_and_no_body() {
+    let router = router_with(
+        settings_with(&format!(
+            r#"
+            [robots-txt]
+            modules = "refuse_all"
+            {HMAC_BLOCK}
+            {PROBE_BLOCK}
+            "#
+        )),
+        &[seam_probe::builder()],
+    );
+
+    let response = send(&router, Method::HEAD, "/robots.txt");
+
+    assert_eq!(response.status().as_u16(), 200);
+    assert_eq!(
+        header_text(&response, "content-type"),
+        Some("text/plain; charset=utf-8")
+    );
+    assert_eq!(
+        header_text(&response, "cache-control"),
+        Some("public, max-age=3600")
+    );
+    assert!(body_text(response).is_empty(), "a HEAD carries no body");
+}
+
+/// The id of the module that answers once.
+const ANSWERS_ONCE_ID: &str = "robots-txt-answers-once";
+
+/// A robots.txt contributor that answers the first time it is asked and fails
+/// every time after, whose answer is due for a refresh at once.
+#[derive(Debug, Default)]
+struct AnswersOnce {
+    asked: std::sync::atomic::AtomicUsize,
+}
+
+#[async_trait::async_trait(?Send)]
+impl RobotsTxtContributor for AnswersOnce {
+    fn refresh(&self) -> std::time::Duration {
+        std::time::Duration::ZERO
+    }
+
+    fn fingerprint(&self) -> String {
+        "answers-once".to_owned()
+    }
+
+    async fn contribute(
+        &self,
+        _call: ModuleCall<'_>,
+    ) -> Result<Contribution, Report<TrustedServerError>> {
+        if self.asked.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
+            Ok(Contribution::parse("User-Agent: OnceBot\nDisallow: /"))
+        } else {
+            Err(Report::new(TrustedServerError::Proxy {
+                message: "the rules service is down".to_owned(),
+            }))
+        }
+    }
+}
+
+fn register_answers_once(
+    _settings: &Settings,
+) -> Result<Option<IntegrationRegistration>, Report<TrustedServerError>> {
+    Ok(Some(
+        IntegrationRegistration::builder(ANSWERS_ONCE_ID)
+            .with_robots_txt_contributor(
+                "robots-txt.answers-once",
+                Arc::new(AnswersOnce::default()),
+            )
+            .build(),
+    ))
+}
+
+fn accept_answers_once(_settings: &Settings) -> Result<bool, Report<TrustedServerError>> {
+    Ok(true)
+}
+
+/// This adapter holds no answer, so every request for the file asks each
+/// module, and a module that fails with nothing held makes the file a `503`
+/// that tells a crawler to come back in ten minutes and is not stored.
+#[test]
+fn with_nothing_held_a_module_that_fails_makes_the_file_unavailable() {
+    let settings = settings_with(&format!(
+        r#"
+        [robots-txt]
+        modules = ["answers-once"]
+        {HMAC_BLOCK}
+
+        [testing]
+        modules = ["robots-txt-answers-once"]
+        "#
+    ));
+    let builder = IntegrationBuilder::new(
+        ANSWERS_ONCE_ID,
+        ANSWERS_ONCE_ID,
+        register_answers_once,
+        accept_answers_once,
+    )
+    .with_module_name("testing.robots-txt-answers-once");
+    let router = router_with(settings, &[builder]);
+
+    let first = get(&router, "/robots.txt", None);
+    assert_eq!(first.status().as_u16(), 200);
+    assert_eq!(body_text(first), "User-Agent: OnceBot\nDisallow: /\n");
+
+    let second = get(&router, "/robots.txt", None);
+    assert_eq!(
+        second.status().as_u16(),
+        503,
+        "nothing is held here, so the module is asked again and its failure fails the file"
+    );
+    assert_eq!(header_text(&second, "retry-after"), Some("600"));
+    assert_eq!(header_text(&second, "cache-control"), Some("no-store"));
+}
+
 /// A request prepares exactly once whether a named route or the fallback
 /// serves it.
 #[test]
diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs
index 28127614cd..80683898e7 100644
--- a/crates/trusted-server-adapter-spin/tests/routes.rs
+++ b/crates/trusted-server-adapter-spin/tests/routes.rs
@@ -1066,3 +1066,238 @@ async fn a_response_carries_no_blanket_rule_without_the_section() {
 
     assert_eq!(robots_tag(&response), None);
 }
+
+/// A module's rules served through this adapter's router, held in the
+/// key-value store the runtime hands the application.
+mod robots_txt_from_a_module {
+    use std::collections::HashMap;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::{Arc, Mutex};
+    use std::time::Duration;
+
+    use bytes::Bytes;
+    use edgezero_core::http::Request;
+    use edgezero_core::key_value_store::{KvError, KvHandle, KvPage, KvStore};
+    use edgezero_core::router::RouterService;
+    use edgezero_core::store_registry::KvRegistry;
+    use error_stack::Report;
+    use trusted_server_adapter_spin::app::TrustedServerApp;
+    use trusted_server_core::error::TrustedServerError;
+    use trusted_server_core::integrations::{IntegrationBuilder, IntegrationRegistration};
+    use trusted_server_core::module_context::ModuleCall;
+    use trusted_server_core::robots_txt::{Contribution, RobotsTxtContributor};
+    use trusted_server_core::settings::Settings;
+
+    use super::robots_request;
+
+    /// The id of the module that answers once.
+    const ANSWERS_ONCE_ID: &str = "robots-txt-answers-once";
+
+    /// A robots.txt contributor that answers the first time it is asked and
+    /// fails every time after, whose answer is due for a refresh at once.
+    #[derive(Debug, Default)]
+    struct AnswersOnce {
+        asked: AtomicUsize,
+    }
+
+    #[async_trait::async_trait(?Send)]
+    impl RobotsTxtContributor for AnswersOnce {
+        fn refresh(&self) -> Duration {
+            Duration::ZERO
+        }
+
+        fn fingerprint(&self) -> String {
+            "answers-once".to_owned()
+        }
+
+        async fn contribute(
+            &self,
+            _call: ModuleCall<'_>,
+        ) -> Result<Contribution, Report<TrustedServerError>> {
+            if self.asked.fetch_add(1, Ordering::SeqCst) == 0 {
+                Ok(Contribution::parse("User-Agent: OnceBot\nDisallow: /"))
+            } else {
+                Err(Report::new(TrustedServerError::Proxy {
+                    message: "the rules service is down".to_owned(),
+                }))
+            }
+        }
+    }
+
+    fn register_answers_once(
+        _settings: &Settings,
+    ) -> Result<Option<IntegrationRegistration>, Report<TrustedServerError>> {
+        Ok(Some(
+            IntegrationRegistration::builder(ANSWERS_ONCE_ID)
+                .with_robots_txt_contributor(
+                    "robots-txt.answers-once",
+                    Arc::new(AnswersOnce::default()),
+                )
+                .build(),
+        ))
+    }
+
+    fn accept_answers_once(_settings: &Settings) -> Result<bool, Report<TrustedServerError>> {
+        Ok(true)
+    }
+
+    /// The route test settings with a file made by the module that answers
+    /// once.
+    fn router_with_a_module_that_answers_once() -> RouterService {
+        let settings = Settings::from_toml(
+            r#"
+                [publisher]
+                domain = "test-publisher.example.com"
+                cookie_domain = ".test-publisher.example.com"
+                origin_url = "https://origin.test-publisher.example.com"
+                proxy_secret = "route-test-proxy-secret"
+
+                [ec]
+                module = "hmac"
+
+                [ec.hmac]
+                passphrase = "test-secret-key-32-bytes-minimum"
+
+                [geo]
+                assume_single_jurisdiction = true
+
+                [robots-txt]
+                modules = ["answers-once"]
+
+                [testing]
+                modules = ["robots-txt-answers-once"]
+            "#,
+        )
+        .expect("should parse route test settings");
+        let builder = IntegrationBuilder::new(
+            ANSWERS_ONCE_ID,
+            ANSWERS_ONCE_ID,
+            register_answers_once,
+            accept_answers_once,
+        )
+        .with_module_name("testing.robots-txt-answers-once");
+
+        TrustedServerApp::routes_with_registrations(settings, &[builder])
+            .expect("should build router from test settings")
+    }
+
+    /// A key-value store kept in memory for one test, so what one request
+    /// writes is what the next one reads.
+    #[derive(Default)]
+    struct MemoryStore {
+        entries: Mutex<HashMap<String, Bytes>>,
+    }
+
+    #[async_trait::async_trait(?Send)]
+    impl KvStore for MemoryStore {
+        async fn delete(&self, key: &str) -> Result<(), KvError> {
+            self.entries
+                .lock()
+                .expect("should lock the store")
+                .remove(key);
+            Ok(())
+        }
+
+        async fn get_bytes(&self, key: &str) -> Result<Option<Bytes>, KvError> {
+            Ok(self
+                .entries
+                .lock()
+                .expect("should lock the store")
+                .get(key)
+                .cloned())
+        }
+
+        async fn list_keys_page(
+            &self,
+            _prefix: &str,
+            _cursor: Option<&str>,
+            _limit: usize,
+        ) -> Result<KvPage, KvError> {
+            Ok(KvPage::default())
+        }
+
+        async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> {
+            self.entries
+                .lock()
+                .expect("should lock the store")
+                .insert(key.to_owned(), value);
+            Ok(())
+        }
+
+        async fn put_bytes_with_ttl(
+            &self,
+            key: &str,
+            value: Bytes,
+            _ttl: Duration,
+        ) -> Result<(), KvError> {
+            self.put_bytes(key, value).await
+        }
+    }
+
+    /// A crawler's `GET` for the file, carrying `store` as the key-value store
+    /// the runtime hands the application, when there is one.
+    fn robots_get(store: Option<&Arc<MemoryStore>>) -> Request {
+        let mut request = robots_request("GET", "/robots.txt");
+        if let Some(store) = store {
+            let handle = KvHandle::new(Arc::clone(store) as Arc<dyn KvStore>);
+            request.extensions_mut().insert(KvRegistry::single_id(
+                "trusted_server_kv".to_owned(),
+                handle,
+            ));
+        }
+        request
+    }
+
+    /// With the store the runtime hands the application, a module's answer
+    /// is held, so when the module next fails the file is made from the
+    /// answer it gave before.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn a_held_answer_is_served_when_its_module_fails() {
+        let router = router_with_a_module_that_answers_once();
+        let store = Arc::new(MemoryStore::default());
+
+        for asked in ["answers", "fails"] {
+            let response = router
+                .oneshot(robots_get(Some(&store)))
+                .await
+                .expect("should route request");
+
+            assert_eq!(response.status().as_u16(), 200, "the module {asked}");
+            let body = response.into_body().into_bytes().unwrap_or_default();
+            assert_eq!(
+                &body[..],
+                b"User-Agent: OnceBot\nDisallow: /\n",
+                "the module {asked}"
+            );
+        }
+    }
+
+    /// With no store from the runtime nothing is held, so a module that fails
+    /// makes the file a `503` that tells a crawler to come back in ten
+    /// minutes and is not stored.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn with_nothing_held_a_module_that_fails_makes_the_file_unavailable() {
+        let router = router_with_a_module_that_answers_once();
+
+        let first = router
+            .oneshot(robots_get(None))
+            .await
+            .expect("should route request");
+        assert_eq!(first.status().as_u16(), 200);
+
+        let second = router
+            .oneshot(robots_get(None))
+            .await
+            .expect("should route request");
+        assert_eq!(second.status().as_u16(), 503);
+        let header = |name: &str| {
+            second
+                .headers()
+                .get(name)
+                .and_then(|value| value.to_str().ok())
+                .map(str::to_owned)
+        };
+        assert_eq!(header("retry-after").as_deref(), Some("600"));
+        assert_eq!(header("cache-control").as_deref(), Some("no-store"));
+    }
+}
diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs
index f3f8a21ab4..43fd30cc2c 100644
--- a/crates/trusted-server-core/src/robots_txt/mod.rs
+++ b/crates/trusted-server-core/src/robots_txt/mod.rs
@@ -659,7 +659,11 @@ mod tests {
             StatusCode::SERVICE_UNAVAILABLE,
             "a crawler reads an empty or allow everything file as permission"
         );
-        assert!(response.headers().contains_key(header::RETRY_AFTER));
+        assert_eq!(
+            response.headers().get(header::RETRY_AFTER),
+            Some(&HeaderValue::from_static("600")),
+            "a crawler should be told to come back in ten minutes"
+        );
         assert_eq!(
             response.headers().get(header::CACHE_CONTROL),
             Some(&HeaderValue::from_static("no-store")),

From f8dc3751d9ace789b05c47c477f4a3def3bb00aa Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 16 Oct 2026 11:35:39 +0100
Subject: [PATCH 12/15] The changelog and guides describe the robots.txt fixes

The changelog and the configuration guide said a module's answer is held in
the key-value store without saying that Fastly and Axum hand the file none,
so there nothing is held, each request asks every module and one failure is
a 503. They now say so, and name the host a module is asked for, that it is
given nothing of the crawler's connection, that a file with no rule is a
503, and that Fastly answers the file before the Edge Cookie steps and the
request filters. The API reference and the integration guide say the same
in their own places.
---
 CHANGELOG.md                    |  2 +-
 docs/guide/api-reference.md     |  4 ++--
 docs/guide/configuration.md     |  5 +++--
 docs/guide/integration-guide.md | 13 +++++++------
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fca8eca57b..473301c408 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -79,7 +79,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- `[robots-txt]` serves a publisher's `/robots.txt` from the settings. `modules` names what makes the file, in order. `refuse_all` and `allow_all` are Trusted Server's own, and any other name is a module that contributes rules through the new `RobotsTxtContributor` extension point, declared with `with_robots_txt_contributor`. Trusted Server writes the file, so a contributor cannot produce a malformed one, a refusal cannot be loosened, and a path in `always_allow` stays open to every crawler refused the whole site. With `refuse_all` selected every response an adapter finalizes carries `X-Robots-Tag: noindex, nofollow`. A module's answer is held in the key-value store until its refresh is due, a module that cannot answer has its last answer served, and with none held the answer is `503` and never an empty file. Without the section `/robots.txt` reaches the publisher's origin as before.
+- `[robots-txt]` serves a publisher's `/robots.txt` from the settings. `modules` names what makes the file, in order. `refuse_all` and `allow_all` are Trusted Server's own, and any other name is a module that contributes rules through the new `RobotsTxtContributor` extension point, declared with `with_robots_txt_contributor`. Trusted Server writes the file, so a contributor cannot produce a malformed one, a refusal cannot be loosened, and a path in `always_allow` stays open to every crawler refused the whole site. With `refuse_all` selected every response an adapter finalizes carries `X-Robots-Tag: noindex, nofollow`. A module is asked for the host the request was addressed to, never one a forwarded header names, and is given nothing of the crawler's connection. Its answer is held for that host in the key-value store until its refresh is due on Cloudflare and Spin, while Fastly and Axum hold nothing and ask each module on every request. A module that cannot answer has its last held answer served, and with none held, or with no rule in the whole file, the answer is `503` and never an empty file. On Fastly the file is answered before the Edge Cookie steps and the request filters. Without the section `/robots.txt` reaches the publisher's origin as before.
 - `ts keys rotate` and `ts keys deactivate` rotate and retire the request signing keys from an operator's machine. They write the two stores the service links as `jwks_store` and `signing_keys` through the `fastly` CLI, with the token from `FASTLY_API_TOKEN`. A private key reaches `fastly` on its standard input and never in its arguments, and a key id no platform can store is refused before anything is written. A command that cannot read the config store writes nothing, and a delete that failed part way can be run again.
 - Attestation. With `[attestation]` in the settings a deployment serves evidence, in the sense of RFC 9334, of who operates it and which build it runs, as a page and as JSON at `[attestation] endpoint`, which defaults to `/_ts/attestation`. The evidence names the host the request arrived for, the publisher, the platform service and the build, with the time and the caller's nonce, and is signed with ECDSA P-256 under a configured context. The signing keys are a schedule compiled in from the build input `TRUSTED_SERVER_ATTESTATION_KEYS`, and a build given none answers `503` at the endpoint. A build also reports the commit, the build run, the tag it was made from and the build time it was given, from `TRUSTED_SERVER_COMMIT`, `TRUSTED_SERVER_BUILD_RUN`, `TRUSTED_SERVER_BUILD_TAG` and `SOURCE_DATE_EPOCH`. See `docs/guide/attestation.md`.
 - One Fastly service can serve several publishers. A `__KEY` selector that contains `{host}` names an app-config blob for each host, pushed with `ts config push --key <host>`, and each request reads the blob under its own host. A host with no blob is answered `421 Misdirected Request`, never from the logical store ID or from another host's blob, and a sandbox that serves several requests keeps an application for each blob, found only by that blob's key. A selector without the placeholder reads one blob as before. `scripts/config-by-host-local-test.sh` runs it under Viceroy.
diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md
index eb5e7ac9e6..4f5e08d6b4 100644
--- a/docs/guide/api-reference.md
+++ b/docs/guide/api-reference.md
@@ -1046,8 +1046,8 @@ a publisher keeps their own file. `HEAD` is answered with the headers of the
   `Cache-Control: public, max-age=3600`
 - `503 Service Unavailable` with `Retry-After: 600` and
   `Cache-Control: no-store` when a selected module cannot answer and no earlier
-  answer of its is held, because an empty file or a `404` reads to a crawler as
-  permission to crawl everything
+  answer of its is held, or when the file would hold no rule, because an empty
+  file or a `404` reads to a crawler as permission to crawl everything
 
 With `refuse_all` selected the file refuses every crawler, and every response
 the deployment finalizes also carries `X-Robots-Tag: noindex, nofollow`.
diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md
index 426d381a7b..20ebaf64bc 100644
--- a/docs/guide/configuration.md
+++ b/docs/guide/configuration.md
@@ -1054,9 +1054,10 @@ A module with settings of its own reads them from its own block, `[robots-txt.<n
 1. With `refuse_all` selected the file is the refusal and nothing else, whatever else is selected, and every response the deployment finalizes carries `X-Robots-Tag: noindex, nofollow`.
 2. A path in `always_allow` stays open to every crawler a module's rules refuse the whole site to, `*` included. A narrower rule is kept as the module gave it, so `Disallow: /ads` still closes off `/ads.txt`.
 3. The rules appear in the order `modules` names them, between `top_text` above and the `Sitemap` line and `bottom_text` below.
-4. A module's answer is held in the key-value store and served until the module's own refresh is due. A module that cannot answer has its last answer served past its age, for as long as the store keeps it, which is 30 days where the store can give an entry a lifetime. With no answer held at all the file is `503` with `Retry-After`, never an empty file, which a crawler would read as permission to crawl everything.
+4. A module is asked for the host the request was addressed to, never one that `Forwarded` or `X-Forwarded-Host` names, and is given nothing of the crawler's connection. Its answer is held for that host in the key-value store and served until the module's own refresh is due. A module that cannot answer has its last answer served past its age, for as long as the store keeps it, which is 30 days where the store can give an entry a lifetime. With no answer held at all, or with no rule anywhere in the file, the file is `503` with `Retry-After: 600`, never an empty file, which a crawler would read as permission to crawl everything. Select `allow_all` to allow every crawler.
 5. Spin's key-value store gives an entry no lifetime, so there an answer is kept until it is replaced, and an answer held for settings that have since changed stays until it is deleted from the store.
-6. A deployment whose key-value store cannot be opened holds nothing, so there every request for the file asks each module.
+6. Fastly and Axum hold nothing, because neither hands the file a key-value store, and neither does a deployment whose store cannot be opened. There every request for the file asks each module, and a module that cannot answer makes the file `503`.
+7. On Fastly the file is answered once the modules have prepared the request, before the Edge Cookie steps and the request filters, so a filter's answer never stands in for it.
 
 **Which responses carry the header.** An adapter writes `X-Robots-Tag` last when it finalizes a response, replacing any tag already there, so neither an origin page nor `[response_headers]` can loosen it. That covers every route, the publisher fallback and the error path. A response sent before that step does not carry it. On every adapter but Fastly that is the `404` or `405` the router gives a request it does not route. On Fastly it is the two debug endpoints under `/_ts/debug` and a failure before the settings are read.
 
diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md
index ac3fdabaac..4fd7d79d93 100644
--- a/docs/guide/integration-guide.md
+++ b/docs/guide/integration-guide.md
@@ -334,12 +334,13 @@ that supply a module of one type under one name.
 
 A robots.txt contributor implements `RobotsTxtContributor`. It returns groups
 of records and never bytes, so it cannot produce a malformed file. Trusted
-Server asks it for the request without its query, holds the answer until the
-contributor's `refresh` is due, and writes the file from every contributor
-`[robots-txt] modules` names, in that order. `fingerprint` names the settings
-an answer was given for, so an answer held for other settings is never
-served. Trusted Server hashes it into the key the answer is held under, so
-nothing of it is written to the store.
+Server asks it for the request without its query, on the host the request was
+addressed to and with no client information or host signals in its services,
+holds the answer for that host until the contributor's `refresh` is due, and
+writes the file from every contributor `[robots-txt] modules` names, in that
+order. `fingerprint` names the settings an answer was given for, so an answer
+held for other settings is never served. Trusted Server hashes it into the key
+the answer is held under, so nothing of it is written to the store.
 
 The Fastly adapter is the one adapter that classifies a request, so a device
 module is asked there and on no other adapter. It is shown the User-Agent and

From 86c07b83ec38f60deee5879c79efabb8c46b7884 Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Fri, 16 Oct 2026 11:39:44 +0100
Subject: [PATCH 13/15] Say what the addressed host and the held key are made
 of

---
 crates/trusted-server-core/src/http_util.rs          | 5 +++--
 crates/trusted-server-core/src/robots_txt/holding.rs | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs
index f2c528045f..73c1513fe0 100644
--- a/crates/trusted-server-core/src/http_util.rs
+++ b/crates/trusted-server-core/src/http_util.rs
@@ -169,8 +169,9 @@ pub(crate) fn request_host(req: &Request<EdgeBody>) -> &str {
 ///
 /// Read from `Host`, or from the URI's host when there is no `Host` header,
 /// and never from `Forwarded` or `X-Forwarded-Host`, which a client can set to
-/// any host it likes. What is signed for a host, or held for everyone who
-/// asks it, is keyed on this, so one request cannot claim another host.
+/// any host it likes. What is signed for a host, or held for every request to
+/// it, is keyed on this, so a forwarded header cannot put another host in its
+/// place.
 pub(crate) fn addressed_host(req: &Request<EdgeBody>) -> String {
     let raw = req
         .headers()
diff --git a/crates/trusted-server-core/src/robots_txt/holding.rs b/crates/trusted-server-core/src/robots_txt/holding.rs
index 66e3727b08..d85500d0ca 100644
--- a/crates/trusted-server-core/src/robots_txt/holding.rs
+++ b/crates/trusted-server-core/src/robots_txt/holding.rs
@@ -49,8 +49,8 @@ struct Held {
     contribution: Contribution,
 }
 
-/// Who an answer is for and who gave it, which together name where it is
-/// held.
+/// Who an answer is for and who gave it, which with the host the contributor
+/// is asked for name where it is held.
 pub(crate) struct Holder<'a> {
     /// The publisher the file is for. One store may serve several, and one
     /// publisher's rules are never another's.

From c03143d757ead2020146e5d3a25d9923b9487fbb Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Sat, 17 Oct 2026 10:58:21 +0100
Subject: [PATCH 14/15] The changelog and guide say no adapter holds a
 robots.txt answer here

The changelog and the configuration guide said a module's answer is held
in the key-value store on Cloudflare and Spin while Fastly and Axum hold
nothing. On this branch no adapter gives a request a working key-value
store. Fastly and Axum give UnavailableKvStore, and the Cloudflare and Spin
adapters declare no store in Hooks::stores, so EdgeZero's run_app opens
none and their requests get UnavailableKvStore too. Nothing is held on any
adapter, every request asks each module, and a module that cannot answer
makes the file 503. The guide's point about Spin's store and lifetimes
described a store the file is never given, so it goes.
---
 CHANGELOG.md                | 2 +-
 docs/guide/configuration.md | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 03fd662cdf..e4807c24c3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -82,7 +82,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- `[robots-txt]` serves a publisher's `/robots.txt` from the settings. `modules` names what makes the file, in order. `refuse_all` and `allow_all` are Trusted Server's own, and any other name is a module that contributes rules through the new `RobotsTxtContributor` extension point, declared with `with_robots_txt_contributor`. Trusted Server writes the file, so a contributor cannot produce a malformed one, a refusal cannot be loosened, and a path in `always_allow` stays open to every crawler refused the whole site. With `refuse_all` selected every response an adapter finalizes carries `X-Robots-Tag: noindex, nofollow`. A module is asked for the host the request was addressed to, never one a forwarded header names, and is given nothing of the crawler's connection. Its answer is held for that host in the key-value store until its refresh is due on Cloudflare and Spin, while Fastly and Axum hold nothing and ask each module on every request. A module that cannot answer has its last held answer served, and with none held, or with no rule in the whole file, the answer is `503` and never an empty file. On Fastly the file is answered before the Edge Cookie steps and the request filters. Without the section `/robots.txt` reaches the publisher's origin as before.
+- `[robots-txt]` serves a publisher's `/robots.txt` from the settings. `modules` names what makes the file, in order. `refuse_all` and `allow_all` are Trusted Server's own, and any other name is a module that contributes rules through the new `RobotsTxtContributor` extension point, declared with `with_robots_txt_contributor`. Trusted Server writes the file, so a contributor cannot produce a malformed one, a refusal cannot be loosened, and a path in `always_allow` stays open to every crawler refused the whole site. With `refuse_all` selected every response an adapter finalizes carries `X-Robots-Tag: noindex, nofollow`. A module is asked for the host the request was addressed to, never one a forwarded header names, and is given nothing of the crawler's connection. Its answer is held for that host in the request's key-value store until its refresh is due, and no adapter gives a request a working one, so every adapter asks each module on every request. A module that cannot answer has its last held answer served, and with none held, or with no rule in the whole file, the answer is `503` and never an empty file. On Fastly the file is answered before the Edge Cookie steps and the request filters. Without the section `/robots.txt` reaches the publisher's origin as before.
 - `ts keys rotate` and `ts keys deactivate` rotate and retire the request signing keys from an operator's machine. They write the two stores the service links as `jwks_store` and `signing_keys` through the `fastly` CLI, with the token from `FASTLY_API_TOKEN`. A private key reaches `fastly` on its standard input and never in its arguments, and a key id no platform can store is refused before anything is written. A command that cannot read the config store writes nothing, and a delete that failed part way can be run again.
 - Attestation. With `[attestation]` in the settings a deployment serves evidence, in the sense of RFC 9334, of who operates it and which build it runs, as a page and as JSON at `[attestation] endpoint`, which defaults to `/_ts/attestation`. The evidence names the host the request arrived for, the publisher, the platform service and the build, with the time and the caller's nonce, and is signed with ECDSA P-256 under a configured context. The signing keys are a schedule compiled in from the build input `TRUSTED_SERVER_ATTESTATION_KEYS`, and a build given none answers `503` at the endpoint, as does a clock that cannot be read. A key in the schedule that is not a P-256 private key stops the build. A build also reports the commit, the build run, the tag it was made from and the build time it was given, from `TRUSTED_SERVER_COMMIT`, `TRUSTED_SERVER_BUILD_RUN`, `TRUSTED_SERVER_BUILD_TAG` and `SOURCE_DATE_EPOCH`. See `docs/guide/attestation.md`.
 - One Fastly service can serve several publishers. A `TS__CONFIG__KEY` entry in the `trusted_server_config` store that contains `{host}` names an app-config blob for each host, pushed with `ts config push --key <host>`, and each request reads the blob under its own host. A host with no blob is answered `421 Misdirected Request`, never from the logical store ID or from another host's blob, a malformed selector is answered `500`, and a sandbox that serves several requests keeps an application for each blob, found only by that blob's key. Without the entry, or with a selector that has no placeholder, the service reads one blob. `scripts/config-by-host-local-test.sh` runs it under Viceroy.
diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md
index e87dbd9368..4d32a49c76 100644
--- a/docs/guide/configuration.md
+++ b/docs/guide/configuration.md
@@ -1054,10 +1054,9 @@ A module with settings of its own reads them from its own block, `[robots-txt.<n
 1. With `refuse_all` selected the file is the refusal and nothing else, whatever else is selected, and every response the deployment finalizes carries `X-Robots-Tag: noindex, nofollow`.
 2. A path in `always_allow` stays open to every crawler a module's rules refuse the whole site to, `*` included. A narrower rule is kept as the module gave it, so `Disallow: /ads` still closes off `/ads.txt`.
 3. The rules appear in the order `modules` names them, between `top_text` above and the `Sitemap` line and `bottom_text` below.
-4. A module is asked for the host the request was addressed to, never one that `Forwarded` or `X-Forwarded-Host` names, and is given nothing of the crawler's connection. Its answer is held for that host in the key-value store and served until the module's own refresh is due. A module that cannot answer has its last answer served past its age, for as long as the store keeps it, which is 30 days where the store can give an entry a lifetime. With no answer held at all, or with no rule anywhere in the file, the file is `503` with `Retry-After: 600`, never an empty file, which a crawler would read as permission to crawl everything. Select `allow_all` to allow every crawler.
-5. Spin's key-value store gives an entry no lifetime, so there an answer is kept until it is replaced, and an answer held for settings that have since changed stays until it is deleted from the store.
-6. Fastly and Axum hold nothing, because neither hands the file a key-value store, and neither does a deployment whose store cannot be opened. There every request for the file asks each module, and a module that cannot answer makes the file `503`.
-7. On Fastly the file is answered once the modules have prepared the request, before the Edge Cookie steps and the request filters, so a filter's answer never stands in for it.
+4. A module is asked for the host the request was addressed to, never one that `Forwarded` or `X-Forwarded-Host` names, and is given nothing of the crawler's connection. Its answer is held for that host in the key-value store a request is given, and served until the module's own refresh is due. A module that cannot answer has its last answer served past its age, for as long as the store keeps it, which is 30 days where the store can give an entry a lifetime. With no answer held at all, or with no rule anywhere in the file, the file is `503` with `Retry-After: 600`, never an empty file, which a crawler would read as permission to crawl everything. Select `allow_all` to allow every crawler.
+5. No adapter gives a request a working key-value store, so nothing is held. Fastly and Axum give one whose every call fails, and Cloudflare and Spin declare no store for EdgeZero to open. Every request for the file asks each module, and a module that cannot answer makes the file `503`.
+6. On Fastly the file is answered once the modules have prepared the request, before the Edge Cookie steps and the request filters, so a filter's answer never stands in for it.
 
 **Which responses carry the header.** An adapter writes `X-Robots-Tag` last when it finalizes a response, replacing any tag already there, so neither an origin page nor `[response_headers]` can loosen it. That covers every route, the publisher fallback and the error path. A response sent before that step does not carry it. On every adapter but Fastly that is the `404` or `405` the router gives a request it does not route. On Fastly it is the two debug endpoints under `/_ts/debug` and a failure before the settings are read.
 

From 5e79072fe944bdcc68d1bd82f4c0234e6ce471df Mon Sep 17 00:00:00 2001
From: James Rosewell <james@51degrees.com>
Date: Wed, 28 Oct 2026 11:28:46 +0000
Subject: [PATCH 15/15] The robots.txt route tests carry the sample policy

---
 crates/trusted-server-adapter-cloudflare/tests/routes.rs | 8 ++++++--
 crates/trusted-server-adapter-spin/tests/routes.rs       | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs
index dfb4a4f688..ab07e52833 100644
--- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs
+++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs
@@ -904,7 +904,9 @@ fn router_refusing_every_crawler() -> RouterService {
             always_allow = ["/ads.txt"]
         "#,
     )
-    .expect("should parse route test settings");
+    .expect("should parse route test settings")
+    // A loaded domain carries its policy.
+    .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample());
 
     TrustedServerApp::routes_with_settings(settings)
         .expect("should build router from test settings")
@@ -1135,7 +1137,9 @@ mod robots_txt_from_a_module {
                 modules = ["robots-txt-answers-once"]
             "#,
         )
-        .expect("should parse route test settings");
+        .expect("should parse route test settings")
+        // A loaded domain carries its policy.
+        .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample());
         let builder = IntegrationBuilder::new(
             ANSWERS_ONCE_ID,
             ANSWERS_ONCE_ID,
diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs
index 186d82c5ab..3f686305b6 100644
--- a/crates/trusted-server-adapter-spin/tests/routes.rs
+++ b/crates/trusted-server-adapter-spin/tests/routes.rs
@@ -1238,7 +1238,9 @@ fn router_refusing_every_crawler() -> RouterService {
             always_allow = ["/ads.txt"]
         "#,
     )
-    .expect("should parse route test settings");
+    .expect("should parse route test settings")
+    // A loaded domain carries its policy.
+    .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample());
 
     TrustedServerApp::routes_with_settings(settings)
         .expect("should build router from test settings")
@@ -1469,7 +1471,9 @@ mod robots_txt_from_a_module {
                 modules = ["robots-txt-answers-once"]
             "#,
         )
-        .expect("should parse route test settings");
+        .expect("should parse route test settings")
+        // A loaded domain carries its policy.
+        .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample());
         let builder = IntegrationBuilder::new(
             ANSWERS_ONCE_ID,
             ANSWERS_ONCE_ID,