From a57313ba9077a0904eaf38af066c19830bc726f9 Mon Sep 17 00:00:00 2001
From: Yev Meyer <ymeyer@nvidia.com>
Date: Fri, 30 Oct 2026 19:17:57 -0400
Subject: [PATCH 1/6] docs(engine): list every probe hook and declaration,
 checked by a test

architecture/probes.md lists every public method of BaseProbe, BankBackedProbe and the six mixins in its hook table, with where each is called, and every class attribute and classmethod in a new table, with its default and what reads it. tests/test_reference_docs.py fails when a public name on those classes has no row, when a row names nothing they define, when a method sits outside the hook table, or when a documented default value differs from the class.

BaseProbe defines should_inline_judge_user_turn and should_inline_judge_assistant_turn, and AgenticMixin overrides the first, but nothing calls either. The tables, the authoring guide, the docstrings and the agentic template say so.

Signed-off-by: Yev Meyer <ymeyer@nvidia.com>
---
 architecture/probes.md               |  69 ++++++++++++++++--
 docs/engine/AUTHORING_A_PROBE.md     |   4 +-
 src/usersim/engine/core/probes.py    |  24 +++---
 templates/probe/generator_agentic.py |   4 +-
 tests/test_reference_docs.py         | 105 +++++++++++++++++++++++++++
 5 files changed, 180 insertions(+), 26 deletions(-)
 create mode 100644 tests/test_reference_docs.py

diff --git a/architecture/probes.md b/architecture/probes.md
index 908dc9a..d402545 100644
--- a/architecture/probes.md
+++ b/architecture/probes.md
@@ -30,10 +30,13 @@ Shipped probes are listed by module in `BUILTIN_PROBE_MODULES`
 looked up, which is when their decorators run. Out-of-tree probes are
 imported through the `usersim.probes` entry-point group instead.
 
-## Hooks the loop calls
+## Hooks
 
-`core/simulation.py` calls these on the probe. Only the first two are
-required; the rest have working defaults on `BaseProbe`.
+The conversation loop in `core/simulation.py` calls most of these on the probe;
+the table says where the others are called from. Only the first two are
+required, plus `derive_task` on a `BankBackedProbe` and `execute_tool_call`
+with `ToolExecutionMixin`; the rest have working defaults on `BaseProbe` or the
+mixin that adds them.
 
 | Hook | When |
 |---|---|
@@ -45,6 +48,7 @@ required; the rest have working defaults on `BaseProbe`.
 | `format_followup_user_instructions` | turns after the first |
 | `on_followup_failure` | a follow-up turn fails its gate |
 | `get_tools_for_assistant` | tool-using probes |
+| `transform_assistant_history` | every assistant call: the copy of the history the assistant sees, which may differ from the stored transcript |
 | `after_assistant_turn` | after each assistant reply; may execute tools |
 | `format_assistant_judge_prompt` | in-sim assessment of the assistant turn |
 | `is_capitulation_detected` | pressure probes checking for a flip |
@@ -52,13 +56,32 @@ required; the rest have working defaults on `BaseProbe`.
 | `user_turn_policy` | once per trajectory: whether earlier replies may be summarized for context management, whether the last follow-ups may wrap up, which filter phrases the user may write, which names the script check skips, and a check each generated opening must pass before the gate |
 | `seed_state_metadata` | recording bank/seed provenance on the row |
 | `build_result_extras` | adding probe-specific columns to the trajectory, each declared in the class's `output_columns` |
+| `should_succeed` | the end of the conversation: whether the probe completed its protocol; `False` fails the row |
+| `derive_task` | constructing a `BankBackedProbe`: the bank entry the row runs on |
+| `run_dispatch` | once per row, from the dispatcher: runs the conversation, by default through the loop |
+| `execute_tool_call` | `ToolExecutionMixin`: each tool call the assistant makes |
+| `parse_tool_call` | `ToolExecutionMixin`: reading a tool call's name and arguments |
+| `on_tool_round_complete` | `ToolExecutionMixin`: after each round of tool calls |
+| `on_tool_call_executed` | after each tool call runs: `ToolExecutionMixin` calls it, and a probe that runs tool calls itself must |
+| `set_tool_call_observer` | an external host, to watch the tool calls the probe runs |
+| `should_inline_judge_user_turn` | never (see below) |
+| `should_inline_judge_assistant_turn` | never (see below) |
+
+`should_inline_judge_user_turn` and `should_inline_judge_assistant_turn` are
+defined on `BaseProbe`, but nothing calls them: every generated user turn goes
+through the gate and every assistant turn through the judge, whatever they
+return.
+
+Two helpers are for the probe to call:
+
+| Helper | Gives |
+|---|---|
+| `theme` | the row's theme, one of those `themes` lists |
+| `assets_dir` | the probe's asset folder, `<root>/<label>/`, where its bank loaders read |
 
 Anything a probe does not override inherits a default, so a new probe declares
-only what is specific to it. Its `description`, a paragraph on what it tests, is
-what the per-row preview and the capability report show with its rows, and its
-`preview_fields` are what the preview's Probe Inputs panel shows about the bank
-entry or theme each row drew. Its `default_scorers` are the scorers
-`usersim eval --scorers auto` runs when a run holds its rows.
+only what is specific to it; [What a probe declares](#what-a-probe-declares)
+lists each declaration.
 
 A probe's own columns are the one part of this that is declared twice. Whatever
 `build_result_extras` returns is written only if the name is a core column or
@@ -90,6 +113,36 @@ Six hooks can reach a model, and the loop awaits them: `get_verbatim_first_user_
 `async def`. `usersim.testing.assert_probe_conforms` reports a plain `def` override,
 which would otherwise raise partway through a conversation.
 
+## What a probe declares
+
+Everything else the core needs from a probe it reads from the class: an
+attribute, or a classmethod where the answer depends on the locale or the run's
+settings.
+
+| Declaration | Default | Read by |
+|---|---|---|
+| `label` | none: required | the registry, and every stage that names the probe |
+| `family`, `prompt_version`, `variants` | set by `@register_probe` | the row's `trajectory_id`, provenance and `probe_family` column |
+| `description` | `""` | the per-row preview and the capability report, beside the probe's rows |
+| `preview_fields` | `()` | the per-row preview's Probe Inputs panel |
+| `default_scorers` | `()` | `usersim eval --scorers auto`, for a run that holds the probe's rows |
+| `assets_root` | `None`: the packaged banks | `assets_dir()`, and through it the probe's loaders, the pre-flight and smoke |
+| `locale_bank_check` | `None`: the asset folder holds bank files | the pre-flight, for each run locale |
+| `bank_locale` | an India variant's own bank when it has one, else its base locale's | the pre-flight |
+| `authoring_guide` | `None` | the pre-flight, when a run asks for a locale the probe has no bank for |
+| `asset_problems` | no problems | `usersim smoke` and `assert_probe_conforms` |
+| `themes` | `None`: no theme | run setup, which draws one per row into the `theme` column |
+| `model_aliases` | `()` | run setup, which adds each alias the models file defines to every row's models |
+| `output_columns` | `()` | the run's `side_effect_columns`; a column left out is dropped |
+| `supports_assistant_resampling` | `True`; `False` with the tool mixins | judge-gated assistant resampling (`max_assistant_attempts > 1`) |
+| `bank_loader` | none: required on a `BankBackedProbe` | constructing a `BankBackedProbe`, for the row's locale |
+| `placeholder_warning_kind` | none: required on a `BankBackedProbe` | the warning a row records when its task is a placeholder, which the capability report counts |
+| `bank_version_key` | `None`: one bank per locale | where provenance records the bank's version |
+| `verbatim_field` | `"verbatim_text"` | `BankVerbatimMixin`: the task field turn 1 reads |
+| `turn1_instruction_template` | `""` | `CustomTurn1InstructionMixin`: the instruction turn 1 is written from |
+| `tool_loop_mode` | `"single"` | `ToolExecutionMixin`: one round of tool calls per turn, or with `"multi"` rounds until the assistant stops calling tools |
+| `tool_max_calls_per_turn` | `8` | `ToolExecutionMixin`: the most tool calls it runs in one turn |
+
 ## Mixins
 
 Six mixins cover the shapes that recur, all in `core/probes.py`:
diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md
index 58d3d75..9354823 100644
--- a/docs/engine/AUTHORING_A_PROBE.md
+++ b/docs/engine/AUTHORING_A_PROBE.md
@@ -140,8 +140,8 @@ implementations.
 | `async format_followup_user_instructions(turn_idx, state)` | no | `[]` | When follow-up turns get bank-driven nudges (reframings) |
 | `on_followup_failure(state, error_kind)` | no | `"skip"` | When gate exhaustion should ABORT the trajectory (safety_chat_pressure does this via `BankReframingMixin`) |
 | `should_succeed(state)` | no | `True` | Always: assert your probe's join key is present. A False fails the row, so check the probe's protocol, not the assistant's behavior |
-| `should_inline_judge_user_turn(turn_idx, state)` | no | `True` | Rarely: `AgenticMixin` skips turn-0 |
-| `should_inline_judge_assistant_turn(turn_idx, state)` | no | `True` | Rarely |
+| `should_inline_judge_user_turn(turn_idx, state)` | no | `True` | Never: nothing calls it, so an override has no effect |
+| `should_inline_judge_assistant_turn(turn_idx, state)` | no | `True` | Never: nothing calls it, so an override has no effect |
 | `allow_early_stop_at_turn(turn_idx, state)` | no | `True` | `ToolCallingMixin` defers until tools are used; `safety_chat_pressure` returns False (the dedicated capitulation classifier on the assistant side is the only stop signal) |
 | `async is_capitulation_detected(state)` | no | `False` | `safety_chat_pressure` overrides this and runs a dedicated comparator-shaped classifier (`classifier.py::classify_capitulation`) against the assistant's last response. When True the simulator sets `stopped_on_capitulation=True`, `capitulated_at_turn=N`, emits `EARLY_STOP` + `CAPITULATION_CHECK` traces, and breaks the loop. Capitulation-specific by design: if a future probe needs a different post-assistant-turn stop reason, add a sibling hook with its own dedicated vocabulary rather than overloading this one. |
 | `user_turn_policy()` | no | `UserTurnPolicy()`: the loop's own behavior | When the probe tests what the model says across turns: turn off compression so the model sees its own replies, turn off the wrap-up anchor so every follow-up stays on topic, allow filter phrases its users naturally write, and skip names (such as brand names in Latin script) in the script check. `identity_disclosure` does all four, and checks the openings its users write for AI names. A generated opening can also be checked before the gate (`check_opening`): return why a draft breaks the probe's rules, and the loop asks for another draft within `max_query_attempts`. Read once per trajectory, after `derive_task` |
diff --git a/src/usersim/engine/core/probes.py b/src/usersim/engine/core/probes.py
index fb02433..835b0a9 100644
--- a/src/usersim/engine/core/probes.py
+++ b/src/usersim/engine/core/probes.py
@@ -147,8 +147,8 @@ class ProbeAdapter(Protocol):
     format_followup_user_instructions(turn_idx, state) -> list[str]
     transform_assistant_history(messages, current_user_turn) -> list[dict]
     on_followup_failure(state, error_kind) -> str
-    should_inline_judge_user_turn(turn_idx, state) -> bool
-    should_inline_judge_assistant_turn(turn_idx, state) -> bool
+    should_inline_judge_user_turn(turn_idx, state) -> bool  (not called)
+    should_inline_judge_assistant_turn(turn_idx, state) -> bool  (not called)
     allow_early_stop_at_turn(turn_idx, state) -> bool
     is_capitulation_detected(state) -> bool
     user_turn_policy() -> UserTurnPolicy
@@ -635,19 +635,18 @@ def on_followup_failure(self, state: Any, error_kind: str) -> str:
         return "skip"
 
     def should_inline_judge_user_turn(self, turn_idx: int, state: Any) -> bool:
-        """Default True: every user turn flows through the user-judge gate.
+        """Whether a generated user turn goes through the user-judge gate. Nothing calls it.
 
-        ``AgenticMixin`` returns False for turn 0 (the user's verbatim
-        action request is already auditable; gate would burn judge
-        calls for no signal).
+        The loop gates every generated user turn whatever this returns, so an
+        override has no effect.
         """
         return True
 
     def should_inline_judge_assistant_turn(self, turn_idx: int, state: Any) -> bool:
-        """Default True: every assistant turn flows through the quality judge.
+        """Whether an assistant turn goes through the assistant judge. Nothing calls it.
 
-        Override sparingly — the assistant-quality judge is what
-        feeds ``poor_assistant_response`` warnings.
+        The loop judges every assistant turn whatever this returns, so an
+        override has no effect.
         """
         return True
 
@@ -1099,11 +1098,8 @@ def on_followup_failure(self, state: Any, error_kind: str) -> str:
 class AgenticMixin(BankVerbatimMixin):
     """For probes with a single user turn + multi-step assistant tool loop.
 
-    Sets:
-
-    - ``should_inline_judge_user_turn`` — False for turn 0 (the
-      verbatim action request is auditable as-is; gate burns judge
-      calls for no signal on agentic probes).
+    Turns assistant resampling off. Its ``should_inline_judge_user_turn``
+    returns False for turn 0, but nothing calls that hook.
 
     Subclasses still implement ``after_assistant_turn`` themselves
     to drive the inner tool-interception loop with mock responses
diff --git a/templates/probe/generator_agentic.py b/templates/probe/generator_agentic.py
index 7465462..c50cd63 100644
--- a/templates/probe/generator_agentic.py
+++ b/templates/probe/generator_agentic.py
@@ -22,8 +22,8 @@
 
 What ``AgenticMixin`` does:
 - Inherits from ``BankVerbatimMixin``.
-- Sets ``should_inline_judge_user_turn = False`` for turn 0 (the
-  user's verbatim action request is auditable as-is — no judge call).
+- Turns assistant resampling off, since the tool loop's side effects
+  cannot be replayed.
 
 What you provide:
 - ``derive_task(persona, bank, *, cfg)`` — picks the action_request.
diff --git a/tests/test_reference_docs.py b/tests/test_reference_docs.py
new file mode 100644
index 0000000..10e0486
--- /dev/null
+++ b/tests/test_reference_docs.py
@@ -0,0 +1,105 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""The reference tables in ``architecture/probes.md`` list what the probe classes define.
+
+A hook, declaration or helper added to ``BaseProbe``, ``BankBackedProbe`` or a
+mixin without a row fails here, and so does a row left behind when one is
+removed.
+"""
+
+from __future__ import annotations
+
+import ast
+import inspect
+import re
+from pathlib import Path
+
+from usersim.engine.core import probes
+
+_PROBES_DOC = Path(__file__).resolve().parents[1] / "architecture" / "probes.md"
+
+#: The classes a probe builds on, in the order a name is looked up.
+_PROBE_CLASSES = (
+    probes.BaseProbe,
+    probes.BankBackedProbe,
+    probes.BankVerbatimMixin,
+    probes.BankReframingMixin,
+    probes.CustomTurn1InstructionMixin,
+    probes.ToolCallingMixin,
+    probes.ToolExecutionMixin,
+    probes.AgenticMixin,
+)
+
+_NO_VALUE = object()
+
+
+def _public_members() -> dict[str, object]:
+    """Each public name the probe classes define, with the value of its first definition.
+
+    A declaration with a type and no value, such as ``bank_loader``, has
+    ``_NO_VALUE``.
+    """
+    members: dict[str, object] = {}
+    for cls in _PROBE_CLASSES:
+        for name, value in vars(cls).items():
+            if not name.startswith("_"):
+                members.setdefault(name, value)
+    for cls in _PROBE_CLASSES:
+        for name in inspect.get_annotations(cls):
+            if not name.startswith("_"):
+                members.setdefault(name, _NO_VALUE)
+    return members
+
+
+def _table_rows(heading: str) -> list[list[str]]:
+    """The cells of each row of the table whose first column is headed ``heading``."""
+    lines = _PROBES_DOC.read_text(encoding="utf-8").splitlines()
+    start = next((i for i, line in enumerate(lines) if line.startswith(f"| {heading} |")), None)
+    assert start is not None, f"architecture/probes.md has no table headed {heading!r}"
+    rows = []
+    for line in lines[start + 2 :]:
+        if not line.startswith("|"):
+            break
+        rows.append([cell.strip() for cell in line.strip("|").split(" | ")])
+    return rows
+
+
+def _names(heading: str) -> set[str]:
+    """The names in the first column of the table headed ``heading``."""
+    return {name for row in _table_rows(heading) for name in re.findall(r"`(\w+)`", row[0])}
+
+
+def test_every_public_name_on_the_probe_classes_has_a_row() -> None:
+    documented = _names("Hook") | _names("Declaration") | _names("Helper")
+    members = set(_public_members())
+    assert sorted(members - documented) == [], "add a row for each to architecture/probes.md"
+    assert sorted(documented - members) == [], "these rows name nothing the probe classes define"
+
+
+def test_no_name_has_rows_in_two_tables() -> None:
+    hooks, declarations, helpers = _names("Hook"), _names("Declaration"), _names("Helper")
+    assert hooks & declarations == set()
+    assert hooks & helpers == set()
+    assert declarations & helpers == set()
+
+
+def test_hooks_are_methods_and_declarations_are_not() -> None:
+    members = _public_members()
+    for name in _names("Hook"):
+        assert inspect.isfunction(members[name]) or isinstance(members[name], staticmethod), name
+    for name in _names("Declaration"):
+        assert not inspect.isfunction(members[name]), f"{name} is a method: give it a row in the hook table"
+
+
+def test_each_documented_default_value_matches_the_class() -> None:
+    members = _public_members()
+    checked = 0
+    for cells in _table_rows("Declaration"):
+        names = re.findall(r"`(\w+)`", cells[0])
+        literal = re.match(r"`([^`]+)`", cells[1])
+        if len(names) != 1 or literal is None or isinstance(members[names[0]], classmethod):
+            continue
+        assert members[names[0]] == ast.literal_eval(literal.group(1)), names[0]
+        checked += 1
+    assert checked >= 10, "the declaration table's Default column lost its values"

From 7c815027b01a5ad3c4ba61c622fd627d95b59f4d Mon Sep 17 00:00:00 2001
From: Yev Meyer <ymeyer@nvidia.com>
Date: Sat, 31 Oct 2026 15:33:45 -0400
Subject: [PATCH 2/6] test(engine): limit how often shared code names a probe

tests/test_probe_names_in_shared_code.py counts the built-in probe labels written as string constants in src/usersim/ outside engine/probes/, docstrings excluded, file by file. A file over its allowance in _ALLOWED fails with every occurrence listed, and an allowance above the count fails too, so a label taken out cannot come back.

architecture/probes.md points at the test from the declarations table, and the authoring guide says a scorer registered under its probe label adds its module to the allowances.

Signed-off-by: Yev Meyer <ymeyer@nvidia.com>
---
 architecture/probes.md                   |   5 +-
 docs/engine/AUTHORING_A_PROBE.md         |   8 ++
 tests/test_probe_names_in_shared_code.py | 113 +++++++++++++++++++++++
 3 files changed, 125 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_probe_names_in_shared_code.py

diff --git a/architecture/probes.md b/architecture/probes.md
index d402545..97bbe13 100644
--- a/architecture/probes.md
+++ b/architecture/probes.md
@@ -117,7 +117,10 @@ which would otherwise raise partway through a conversation.
 
 Everything else the core needs from a probe it reads from the class: an
 attribute, or a classmethod where the answer depends on the locale or the run's
-settings.
+settings. Code outside `engine/probes/` writes a probe's label only as often as
+[`tests/test_probe_names_in_shared_code.py`](../tests/test_probe_names_in_shared_code.py)
+allows, file by file, so what a new probe needs from the core goes here rather
+than into a branch on its label.
 
 | Declaration | Default | Read by |
 |---|---|---|
diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md
index 9354823..6ded75b 100644
--- a/docs/engine/AUTHORING_A_PROBE.md
+++ b/docs/engine/AUTHORING_A_PROBE.md
@@ -1161,6 +1161,14 @@ evaluator calls each scorer it runs on every row, so return early for rows from
 other probes, for example when your probe's join key is missing, as the
 `safety_agentic` scorer does with `action_request_id`.
 
+A scorer registered under your probe's label writes that label in a module
+outside `engine/probes/`.
+[`tests/test_probe_names_in_shared_code.py`](../../tests/test_probe_names_in_shared_code.py)
+allows each such file a fixed number of probe labels, so add the new module to
+its `_ALLOWED` in the same change. Anything else the core needs from your probe
+goes on the class: see
+[What a probe declares](../../architecture/probes.md#what-a-probe-declares).
+
 The scorer is its own design exercise (axis definition, rubric
 authoring, threshold gating). See
 [`evaluator/scorers/sov_ai_facts.py`](../../src/usersim/engine/evaluator/scorers/sov_ai_facts.py)
diff --git a/tests/test_probe_names_in_shared_code.py b/tests/test_probe_names_in_shared_code.py
new file mode 100644
index 0000000..9336640
--- /dev/null
+++ b/tests/test_probe_names_in_shared_code.py
@@ -0,0 +1,113 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Shared code names a built-in probe no more often than ``_ALLOWED`` says.
+
+A probe label written as a string in ``src/usersim/`` outside ``engine/probes/``
+ties shared code to one probe, and a probe installed as an extension has no
+such code written for it. What a probe needs from the core it declares on its
+class instead (``architecture/probes.md``, "What a probe declares").
+
+Labels are counted file by file, docstrings excluded. A file over its allowance
+fails, and so does an allowance above the count, so a label taken out cannot
+come back unnoticed. A new built-in probe's own scorer module and its rows in
+``taxonomy/capabilities.py`` write its label, and raise their allowances in the
+same change.
+"""
+
+from __future__ import annotations
+
+import ast
+from collections import Counter
+from pathlib import Path
+
+from usersim.engine.core.probes import BUILTIN_PROBE_MODULES, known_probes, resolve_probe
+
+_SRC = Path(__file__).resolve().parents[1] / "src" / "usersim"
+_PROBE_PACKAGES = _SRC / "engine" / "probes"
+
+#: How many built-in probe labels each file outside ``engine/probes/`` may
+#: write as strings, by path under ``src/usersim/``.
+_ALLOWED: dict[str, int] = {
+    "asset_gen/registry.py": 3,
+    "cli/_pipeline.py": 7,
+    "cli/smoke.py": 2,
+    "engine/core/action_taxonomy.py": 1,
+    "engine/core/agentic_bank.py": 2,
+    "engine/core/clinical_profile_bank.py": 4,
+    "engine/core/fact_bank.py": 1,
+    "engine/core/finance_bank.py": 1,
+    "engine/core/identity_spec.py": 1,
+    "engine/core/pressure_bank.py": 1,
+    "engine/core/preview.py": 3,
+    "engine/core/probing_taxonomy.py": 2,
+    "engine/core/query_bank.py": 1,
+    "engine/evaluator/axes.py": 6,
+    "engine/evaluator/scorers/financial_services.py": 4,
+    "engine/evaluator/scorers/identity_disclosure.py": 1,
+    "engine/evaluator/scorers/safety_agentic.py": 1,
+    "engine/evaluator/scorers/safety_chat_pressure.py": 1,
+    "engine/evaluator/scorers/sov_ai_dynamic.py": 1,
+    "engine/evaluator/scorers/sov_ai_facts.py": 1,
+    "engine/evaluator/scorers/sov_ai_multilingual_parity.py": 1,
+    "reporting/capability_report.py": 1,
+    "reporting/identity.py": 1,
+    "taxonomy/capabilities.py": 41,
+}
+
+
+def _builtin_labels() -> set[str]:
+    return {label for label in known_probes() if resolve_probe(label).__module__ in BUILTIN_PROBE_MODULES}
+
+
+def _docstring_nodes(tree: ast.Module) -> set[int]:
+    nodes = set()
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+            first = node.body[0] if node.body else None
+            if (
+                isinstance(first, ast.Expr)
+                and isinstance(first.value, ast.Constant)
+                and isinstance(first.value.value, str)
+            ):
+                nodes.add(id(first.value))
+    return nodes
+
+
+def _label_uses() -> list[tuple[str, int, str]]:
+    """Every built-in label written as a string outside ``engine/probes/``: path, line, label."""
+    labels = _builtin_labels()
+    uses = []
+    for path in sorted(_SRC.rglob("*.py")):
+        if path.is_relative_to(_PROBE_PACKAGES):
+            continue
+        tree = ast.parse(path.read_text(encoding="utf-8"))
+        docstrings = _docstring_nodes(tree)
+        for node in ast.walk(tree):
+            if (
+                isinstance(node, ast.Constant)
+                and isinstance(node.value, str)
+                and node.value in labels
+                and id(node) not in docstrings
+            ):
+                uses.append((path.relative_to(_SRC).as_posix(), node.lineno, node.value))
+    return sorted(uses)
+
+
+def test_no_file_names_probes_more_often_than_allowed():
+    uses = _label_uses()
+    counts = Counter(path for path, _, _ in uses)
+    over = {path for path, count in counts.items() if count > _ALLOWED.get(path, 0)}
+    found = [f"{path}:{line}: {label!r}" for path, line, label in uses if path in over]
+    assert not over, (
+        "Shared code names a probe more often than allowed. Declare what the core needs on the "
+        "probe class instead (architecture/probes.md, 'What a probe declares'). A new built-in "
+        "probe's own scorer module and capability rows write its label: raise their allowances "
+        "in _ALLOWED in the same change.\n" + "\n".join(found)
+    )
+
+
+def test_each_allowance_matches_the_count():
+    counts = Counter(path for path, _, _ in _label_uses())
+    stale = {path: counts[path] for path, allowed in _ALLOWED.items() if counts[path] < allowed}
+    assert not stale, f"Fewer probe labels than allowed; lower these allowances in _ALLOWED to the counts: {stale}"

From 849e295c14d6e04684010df40c09626a8c782534 Mon Sep 17 00:00:00 2001
From: Yev Meyer <ymeyer@nvidia.com>
Date: Sat, 31 Oct 2026 22:48:53 -0400
Subject: [PATCH 3/6] feat(extension): add usersim.extension, one module to
 import the extension API from

usersim.extension exports 59 names: the probe classes, mixins and register_probe, the conversation and outcome types, the core model aliases and acall_llm, the prompt, locale and tool-call helpers, register_scorer, and the capability, asset-domain and backend types. Each is imported from the module that defines it on first use, so an asset domain or backend importing from it does not load the conversation loop, and usersim --help stays fast.

docs/extension_api.md lists every name in usersim.extension and usersim.testing. tests/test_extension_api.py checks both modules against the page, each name against the module that defines it, the imports a type checker reads against the runtime map, and in a subprocess that importing an asset domain or backend leaves the loop unloaded.

Signed-off-by: Yev Meyer <ymeyer@nvidia.com>
---
 architecture/plugins.md     |   2 +
 docs/extension_api.md       | 145 ++++++++++++++++++++++++
 docs/extensions.md          |   5 +-
 src/usersim/extension.py    | 219 ++++++++++++++++++++++++++++++++++++
 tests/test_extension_api.py |  89 +++++++++++++++
 5 files changed, 458 insertions(+), 2 deletions(-)
 create mode 100644 docs/extension_api.md
 create mode 100644 src/usersim/extension.py
 create mode 100644 tests/test_extension_api.py

diff --git a/architecture/plugins.md b/architecture/plugins.md
index a612744..59ba9a8 100644
--- a/architecture/plugins.md
+++ b/architecture/plugins.md
@@ -43,6 +43,8 @@ claiming the same one.
 
 [`docs/extensions.md`](../docs/extensions.md) builds a whole extension
 package: a probe with its own bank, a scorer, their tests and a CI recipe.
+[`docs/extension_api.md`](../docs/extension_api.md) lists every name an
+extension imports, from `usersim.extension` and `usersim.testing`.
 
 An extension probe declares the columns it writes in its class's
 `output_columns`, and a run keeps them alongside the core columns. Other
diff --git a/docs/extension_api.md b/docs/extension_api.md
new file mode 100644
index 0000000..c1a1ae8
--- /dev/null
+++ b/docs/extension_api.md
@@ -0,0 +1,145 @@
+# Extension API
+
+An extension imports from two modules: `usersim.extension`, for writing probes,
+scorers, capabilities, asset domains and execution backends, and
+`usersim.testing`, for testing them. This page lists every name the two export.
+A probe's hooks and declarations are part of the same contract;
+[`architecture/probes.md`](../architecture/probes.md) lists them, and
+[`architecture/plugins.md`](../architecture/plugins.md) lists the entry-point
+groups an extension registers through.
+
+A host that runs probes from its own loop uses `usersim.engine.external`,
+described in [`docs/engine/EXTERNAL_PROBE_RUNTIME.md`](engine/EXTERNAL_PROBE_RUNTIME.md).
+Other `usersim` modules are internal, and code moves between them; import a
+name on this page from `usersim.extension`, wherever it is defined. These names
+can change in any release as well.
+
+```python
+from usersim.extension import BaseProbe, register_probe, register_scorer
+from usersim.testing import assert_probe_conforms
+```
+
+`usersim.extension` imports each name from the module that defines it the
+first time it is used, so an asset domain or a backend that imports from it
+does not load the conversation loop.
+
+## `usersim.extension`
+
+### Probes
+
+| Name | What it is |
+|---|---|
+| `BaseProbe` | The class every probe subclasses, with a default for each optional hook |
+| `BankBackedProbe` | A probe whose rows each run on an entry from a bank it loads per locale |
+| `BankVerbatimMixin` | Turn 1 is a bank entry's text, sent as written |
+| `BankReframingMixin` | Turn 1 from the bank, and follow-ups that reframe it under pressure |
+| `CustomTurn1InstructionMixin` | Turn 1 written by the user model from the probe's own instruction |
+| `ToolCallingMixin` | The assistant is offered tools, and the conversation cannot end early before it uses one |
+| `ToolExecutionMixin` | The assistant's tool calls run against the probe's mock environment |
+| `AgenticMixin` | One user turn from the bank, then the assistant's tool loop |
+| `register_probe` | The class decorator that sets a probe's family, prompt version and variants and adds it to the registry |
+| `ProbeRegistrationError` | Raised when `register_probe` refuses a class, such as one under a label another class holds |
+| `PreviewField` | One line of the per-row preview's Probe Inputs panel, for a probe's `preview_fields` |
+| `LocalePromptPack` | A probe's prompts by locale; constructing one fails when a locale it is given has no prompt |
+| `BankLoadError` | Raised when a `BankBackedProbe` cannot load its bank |
+| `UserTurnPolicy` | What a probe changes about how the loop writes and checks user turns, returned by its `user_turn_policy` hook |
+
+### Conversations and outcomes
+
+| Name | What it is |
+|---|---|
+| `ConversationState` | The conversation so far, as the hooks receive it, with its messages, metadata and `OutcomeBuilder` |
+| `make_result` | Builds the row a conversation produces, for a probe that writes its own `run_dispatch` |
+| `make_failed` | Builds the row for a conversation that failed |
+| `finalize_completed` | Finishes a conversation that ran to its end, failing it when the probe did not complete its protocol |
+| `OutcomeBuilder` | Collects the row's structured outcome during a conversation |
+| `OutcomeStatus` | Whether a row completed, completed with warnings, or failed |
+| `FailureClass` | How a row failed |
+| `FailureAttribution` | What caused a failure: one of the models, the probe's own logic, or infrastructure |
+| `Provenance` | What a row was built from: persona dataset, prompt and bank versions, code |
+| `WarningKind` | The built-in warning kinds a row can record |
+| `SimulationTrace` | One check or model call during the conversation: its kind, turn, alias, tokens and latency |
+| `TraceKind` | Which check or call a `SimulationTrace` records |
+
+### Models and settings
+
+| Name | What it is |
+|---|---|
+| `MODEL_USER` | The alias of the model playing the user |
+| `MODEL_ASSISTANT` | The alias of the assistant under test |
+| `MODEL_JUDGE` | The alias of the model that checks user turns and judges assistant turns during the conversation |
+| `ConversationSimulatorConfig` | The run's settings, which hooks receive as `cfg` |
+| `acall_llm` | Calls the model behind an alias and returns its reply as a message |
+
+### Prompts
+
+| Name | What it is |
+|---|---|
+| `format_persona_for_prompt` | A persona as a text block for the user model's prompt |
+| `persona_full_name` | A persona's name in the order its own script writes it |
+| `persona_uuid` | A persona's content-hashed identifier |
+| `format_behavioral_profile_for_prompt` | A persona's behavioral profile as a prompt block |
+| `ChatRealism` | Which chat-writing rules `format_behavioral_profile_for_prompt` gives the user |
+| `format_interaction_style_instructions` | The prompt block for a user's interaction style |
+| `format_disclosure_instructions` | The prompt block for a user's disclosure style |
+| `format_conversation_history_for_prompt` | A conversation as a text block for a judge prompt |
+| `format_tools_for_prompt` | Tool definitions as a readable prompt block |
+| `render_prompt` | Fills a template's placeholders from given values, then from the row |
+
+### Locales and languages
+
+| Name | What it is |
+|---|---|
+| `SHIPPED_LOCALES` | The locales User Sim ships |
+| `get_conversation_language` | The language a locale's conversations are in |
+| `expected_language_display` | A locale's language as a label for prompts |
+| `language_instruction` | The prompt block telling a model which language, and for some locales which script, to write in |
+
+### Tool calls
+
+| Name | What it is |
+|---|---|
+| `parse_tool_call` | A tool call's name and arguments |
+| `recover_tool_call` | The arguments of a tool call from a completion, falling back to JSON in the reply's text |
+
+### Scorers
+
+| Name | What it is |
+|---|---|
+| `register_scorer` | Adds an async scorer function to the registry under a name |
+| `ScorerFn` | The type of a scorer: an async function of the row and the models |
+| `ScorerRegistrationError` | Raised when `register_scorer` refuses a name |
+| `is_missing` | Whether a table cell is empty: None, NaN, `pd.NA` or `pd.NaT` |
+| `none_for_missing` | A copy of a row with every empty cell as None |
+
+### Capabilities, asset domains and backends
+
+| Name | What it is |
+|---|---|
+| `CapabilityDefinition` | A row of the capability report and the evidence it is scored from |
+| `EvidenceSource` | One probe, scorer and set of axes a capability reads |
+| `AssetDomain` | A domain `usersim gen-assets` can build banks for |
+| `ExecutionBackend` | Somewhere a run can be submitted, for `--backend` |
+| `JobSpec` | What to run, independent of where |
+| `JobHandle` | A backend's reference to work it accepted |
+| `JobStatus` | Where a submitted job is, and why it stopped if it did |
+
+## `usersim.testing`
+
+These checks run against every built-in probe and scorer as well.
+[`architecture/plugins.md`](../architecture/plugins.md#testing-an-extension)
+describes `usersim.testing.pytest_plugin`, the test harness an extension's
+suite can load.
+
+| Name | What it is |
+|---|---|
+| `assert_probe_conforms` | Fails with every problem found in a probe's class and declarations |
+| `probe_conformance_problems` | The same findings, as a list |
+| `assert_scorer_conforms` | Fails with every problem found in a scorer, optionally scoring a sample row |
+| `scorer_conformance_problems` | The same findings, as a list |
+| `run_episode` | Runs one episode of a probe offline, as `usersim simulate` runs it |
+| `EpisodeRun` | What `run_episode` returns: the row and every prompt sent |
+| `ScriptedModel` | A model stand-in that answers from a script |
+| `assert_hosted_parity` | Fails when a probe run through a host differs from the same probe run alone |
+| `hosted_parity_report` | Runs one episode alone and through a host, and compares the two |
+| `HostedParityReport` | What `hosted_parity_report` returns |
diff --git a/docs/extensions.md b/docs/extensions.md
index a8c1ce1..a10ee17 100644
--- a/docs/extensions.md
+++ b/docs/extensions.md
@@ -7,9 +7,10 @@ probe that asks a cooking assistant questions from its own bank, and a scorer
 for the rows it produces.
 
 [`architecture/plugins.md`](../architecture/plugins.md) describes the entry
-point groups and how discovery behaves, and
+point groups and how discovery behaves,
 [`docs/engine/AUTHORING_A_PROBE.md`](engine/AUTHORING_A_PROBE.md) covers every
-hook a probe can implement.
+hook a probe can implement, and [`extension_api.md`](extension_api.md) lists
+every name the package can import from `usersim`.
 
 ## Layout
 
diff --git a/src/usersim/extension.py b/src/usersim/extension.py
new file mode 100644
index 0000000..f119334
--- /dev/null
+++ b/src/usersim/extension.py
@@ -0,0 +1,219 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""The public API for extending NeMo User Sim.
+
+A probe, a scorer, a capability, an asset domain or an execution backend
+imports what it needs from here, and its tests import from ``usersim.testing``.
+``docs/extension_api.md`` lists every name with what it is for. Other
+``usersim`` modules are internal, and code moves between them; the names here
+are imported from this module wherever they are defined, and can change in any
+release as well.
+
+Each name is imported from the module that defines it the first time it is
+used, so ``from usersim.extension import AssetDomain`` loads the asset-domain
+registry and not the conversation loop. ``usersim --help`` imports every
+installed asset domain and backend to list them, and stays fast.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from usersim.asset_gen.registry import AssetDomain
+    from usersim.engine.config import MODEL_ASSISTANT, MODEL_JUDGE, MODEL_USER, ConversationSimulatorConfig
+    from usersim.engine.core.backends import ExecutionBackend, JobHandle, JobSpec, JobStatus
+    from usersim.engine.core.behavioral import (
+        ChatRealism,
+        format_behavioral_profile_for_prompt,
+        format_disclosure_instructions,
+        format_interaction_style_instructions,
+        get_conversation_language,
+    )
+    from usersim.engine.core.identity import persona_uuid
+    from usersim.engine.core.llm import acall_llm
+    from usersim.engine.core.locale import SHIPPED_LOCALES, expected_language_display
+    from usersim.engine.core.messages import format_conversation_history_for_prompt, format_tools_for_prompt
+    from usersim.engine.core.missing import is_missing, none_for_missing
+    from usersim.engine.core.outcomes import (
+        FailureAttribution,
+        FailureClass,
+        OutcomeBuilder,
+        OutcomeStatus,
+        Provenance,
+        SimulationTrace,
+        TraceKind,
+        WarningKind,
+    )
+    from usersim.engine.core.persona import format_persona_for_prompt, persona_full_name
+    from usersim.engine.core.probes import (
+        AgenticMixin,
+        BankBackedProbe,
+        BankLoadError,
+        BankReframingMixin,
+        BankVerbatimMixin,
+        BaseProbe,
+        CustomTurn1InstructionMixin,
+        LocalePromptPack,
+        PreviewField,
+        ProbeRegistrationError,
+        ToolCallingMixin,
+        ToolExecutionMixin,
+        register_probe,
+    )
+    from usersim.engine.core.prompt_loader import render_prompt
+    from usersim.engine.core.simulation import (
+        ConversationState,
+        finalize_completed,
+        language_instruction,
+        make_failed,
+        make_result,
+    )
+    from usersim.engine.core.tool_calls import parse_tool_call, recover_tool_call
+    from usersim.engine.core.user_turn_policy import UserTurnPolicy
+    from usersim.engine.evaluator.scorers import ScorerFn, ScorerRegistrationError, register_scorer
+    from usersim.taxonomy.capabilities import CapabilityDefinition, EvidenceSource
+
+__all__ = [
+    # Probes
+    "AgenticMixin",
+    "BankBackedProbe",
+    "BankLoadError",
+    "BankReframingMixin",
+    "BankVerbatimMixin",
+    "BaseProbe",
+    "CustomTurn1InstructionMixin",
+    "LocalePromptPack",
+    "PreviewField",
+    "ProbeRegistrationError",
+    "ToolCallingMixin",
+    "ToolExecutionMixin",
+    "UserTurnPolicy",
+    "register_probe",
+    # Conversations and outcomes
+    "ConversationState",
+    "FailureAttribution",
+    "FailureClass",
+    "OutcomeBuilder",
+    "OutcomeStatus",
+    "Provenance",
+    "SimulationTrace",
+    "TraceKind",
+    "WarningKind",
+    "finalize_completed",
+    "make_failed",
+    "make_result",
+    # Models and settings
+    "MODEL_ASSISTANT",
+    "MODEL_JUDGE",
+    "MODEL_USER",
+    "ConversationSimulatorConfig",
+    "acall_llm",
+    # Prompts
+    "ChatRealism",
+    "format_behavioral_profile_for_prompt",
+    "format_conversation_history_for_prompt",
+    "format_disclosure_instructions",
+    "format_interaction_style_instructions",
+    "format_persona_for_prompt",
+    "format_tools_for_prompt",
+    "persona_full_name",
+    "persona_uuid",
+    "render_prompt",
+    # Locales and languages
+    "SHIPPED_LOCALES",
+    "expected_language_display",
+    "get_conversation_language",
+    "language_instruction",
+    # Tool calls
+    "parse_tool_call",
+    "recover_tool_call",
+    # Scorers
+    "ScorerFn",
+    "ScorerRegistrationError",
+    "is_missing",
+    "none_for_missing",
+    "register_scorer",
+    # Capabilities, asset domains and backends
+    "AssetDomain",
+    "CapabilityDefinition",
+    "EvidenceSource",
+    "ExecutionBackend",
+    "JobHandle",
+    "JobSpec",
+    "JobStatus",
+]
+
+#: The module each public name is defined in, by module.
+_EXPORTS: dict[str, tuple[str, ...]] = {
+    "usersim.asset_gen.registry": ("AssetDomain",),
+    "usersim.engine.config": ("MODEL_ASSISTANT", "MODEL_JUDGE", "MODEL_USER", "ConversationSimulatorConfig"),
+    "usersim.engine.core.backends": ("ExecutionBackend", "JobHandle", "JobSpec", "JobStatus"),
+    "usersim.engine.core.behavioral": (
+        "ChatRealism",
+        "format_behavioral_profile_for_prompt",
+        "format_disclosure_instructions",
+        "format_interaction_style_instructions",
+        "get_conversation_language",
+    ),
+    "usersim.engine.core.identity": ("persona_uuid",),
+    "usersim.engine.core.llm": ("acall_llm",),
+    "usersim.engine.core.locale": ("SHIPPED_LOCALES", "expected_language_display"),
+    "usersim.engine.core.messages": ("format_conversation_history_for_prompt", "format_tools_for_prompt"),
+    "usersim.engine.core.missing": ("is_missing", "none_for_missing"),
+    "usersim.engine.core.outcomes": (
+        "FailureAttribution",
+        "FailureClass",
+        "OutcomeBuilder",
+        "OutcomeStatus",
+        "Provenance",
+        "SimulationTrace",
+        "TraceKind",
+        "WarningKind",
+    ),
+    "usersim.engine.core.persona": ("format_persona_for_prompt", "persona_full_name"),
+    "usersim.engine.core.probes": (
+        "AgenticMixin",
+        "BankBackedProbe",
+        "BankLoadError",
+        "BankReframingMixin",
+        "BankVerbatimMixin",
+        "BaseProbe",
+        "CustomTurn1InstructionMixin",
+        "LocalePromptPack",
+        "PreviewField",
+        "ProbeRegistrationError",
+        "ToolCallingMixin",
+        "ToolExecutionMixin",
+        "register_probe",
+    ),
+    "usersim.engine.core.prompt_loader": ("render_prompt",),
+    "usersim.engine.core.simulation": (
+        "ConversationState",
+        "finalize_completed",
+        "language_instruction",
+        "make_failed",
+        "make_result",
+    ),
+    "usersim.engine.core.tool_calls": ("parse_tool_call", "recover_tool_call"),
+    "usersim.engine.core.user_turn_policy": ("UserTurnPolicy",),
+    "usersim.engine.evaluator.scorers": ("ScorerFn", "ScorerRegistrationError", "register_scorer"),
+    "usersim.taxonomy.capabilities": ("CapabilityDefinition", "EvidenceSource"),
+}
+
+_MODULE_OF = {name: module for module, names in _EXPORTS.items() for name in names}
+
+
+def __getattr__(name: str) -> Any:
+    module = _MODULE_OF.get(name)
+    if module is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    value = getattr(importlib.import_module(module), name)
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> list[str]:
+    return sorted({*globals(), *__all__})
diff --git a/tests/test_extension_api.py b/tests/test_extension_api.py
new file mode 100644
index 0000000..653663e
--- /dev/null
+++ b/tests/test_extension_api.py
@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""``usersim.extension`` and ``usersim.testing`` export what ``docs/extension_api.md`` lists.
+
+A name added to either module's ``__all__`` without a row on the page fails
+here, and so does a row naming something neither exports. ``usersim.extension``
+imports each name on first use, so the tests also check that every name
+resolves to the object its home module defines and that the imports a type
+checker reads name the same objects.
+"""
+
+from __future__ import annotations
+
+import ast
+import importlib
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+import usersim.extension
+import usersim.testing
+
+_REPO = Path(__file__).resolve().parents[1]
+_REFERENCE = _REPO / "docs" / "extension_api.md"
+
+
+def _documented(module: str) -> list[str]:
+    """The names in the first column of the tables under ``## `module```."""
+    names, inside = [], False
+    for line in _REFERENCE.read_text(encoding="utf-8").splitlines():
+        if line.startswith("## "):
+            inside = line == f"## `{module}`"
+        elif inside and line.startswith("| `"):
+            names.append(line.split("`")[1])
+    return names
+
+
+@pytest.mark.parametrize("module", [usersim.extension, usersim.testing], ids=lambda m: m.__name__)
+def test_the_reference_lists_every_exported_name_once(module):
+    documented = _documented(module.__name__)
+    assert len(documented) == len(set(documented)), "a name has two rows"
+    assert len(module.__all__) == len(set(module.__all__)), "a name is in __all__ twice"
+    unlisted = sorted(set(module.__all__) - set(documented))
+    unexported = sorted(set(documented) - set(module.__all__))
+    assert not unlisted, f"add a row to docs/extension_api.md for: {unlisted}"
+    assert not unexported, f"docs/extension_api.md has a row for a name {module.__name__} does not export: {unexported}"
+
+
+def test_each_name_is_the_object_its_home_module_defines():
+    for module, names in usersim.extension._EXPORTS.items():
+        home = importlib.import_module(module)
+        for name in names:
+            assert getattr(usersim.extension, name) is getattr(home, name), name
+    mapped, listed = set(usersim.extension._MODULE_OF), set(usersim.extension.__all__)
+    assert mapped == listed, f"in __all__ only: {sorted(listed - mapped)}; in _EXPORTS only: {sorted(mapped - listed)}"
+
+
+def test_an_unknown_name_raises_attribute_error():
+    with pytest.raises(AttributeError, match="no_such_name"):
+        usersim.extension.no_such_name
+
+
+def test_the_type_checking_imports_match_the_exports():
+    tree = ast.parse(Path(usersim.extension.__file__).read_text(encoding="utf-8"))
+    (block,) = [
+        node
+        for node in tree.body
+        if isinstance(node, ast.If) and isinstance(node.test, ast.Name) and node.test.id == "TYPE_CHECKING"
+    ]
+    imported = {(node.module, alias.name) for node in block.body for alias in node.names}
+    exported = {(module, name) for module, names in usersim.extension._EXPORTS.items() for name in names}
+    assert imported == exported, (
+        f"in the TYPE_CHECKING imports only: {sorted(imported - exported)}; "
+        f"in _EXPORTS only: {sorted(exported - imported)}"
+    )
+
+
+def test_an_asset_domain_or_backend_import_leaves_the_conversation_loop_unloaded():
+    code = (
+        "import sys\n"
+        "from usersim.extension import AssetDomain, CapabilityDefinition, ExecutionBackend, JobSpec\n"
+        "print(sorted(m for m in ('usersim.engine.core.simulation', 'data_designer') if m in sys.modules))\n"
+    )
+    result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
+    assert result.returncode == 0, result.stderr
+    assert result.stdout.strip() == "[]", f"loaded by the import: {result.stdout.strip()}"

From 4901764bbc5b4f1a28eb4202c8e6a6ef30aa6a02 Mon Sep 17 00:00:00 2001
From: Yev Meyer <ymeyer@nvidia.com>
Date: Sun, 1 Nov 2026 14:59:17 -0500
Subject: [PATCH 4/6] docs(extension): import from usersim.extension in the
 examples

docs/extensions.md, architecture/plugins.md, the authoring guide, the probe templates and the make check-extensions fixture import from usersim.extension and usersim.testing, and every import in them is absolute, as ruff requires. CONTRIBUTING.md names the two modules and links the page that lists their names. The guide's worked example loads its bank through the probe's assets_dir(), from a folder named for its label, and the test template checks registration with assert_probe_conforms.

tests/test_extension_api.py fails when one of these examples imports another usersim module, apart from a listed few: the modules of the in-tree example, the test template's standard tests, which patch the loop's model calls, and the fixture's probe built on identity_disclosure.

Signed-off-by: Yev Meyer <ymeyer@nvidia.com>
---
 CONTRIBUTING.md                               |  4 +-
 architecture/plugins.md                       |  4 +-
 docs/engine/AUTHORING_A_PROBE.md              | 42 +++++-----
 docs/extensions.md                            |  7 +-
 templates/probe/README.md                     |  2 +-
 templates/probe/generator_agentic.py          | 12 +--
 templates/probe/generator_bank_reframing.py   |  4 +-
 templates/probe/generator_bank_verbatim.py    | 12 +--
 templates/probe/generator_baseprobe_only.py   |  7 +-
 templates/probe/generator_custom_turn1.py     |  3 +-
 templates/probe/generator_tool_calling.py     |  2 +-
 templates/probe/prompts.py                    |  3 +-
 templates/probe/test_probe.py.template        | 25 ++----
 .../usersim_extension_fixture/backend.py      |  2 +-
 .../usersim_extension_fixture/capability.py   |  2 +-
 .../usersim_extension_fixture/domain.py       |  2 +-
 .../usersim_extension_fixture/identity.py     |  2 +-
 .../usersim_extension_fixture/probe.py        |  2 +-
 .../usersim_extension_fixture/scorer.py       |  2 +-
 tests/test_extension_api.py                   | 84 +++++++++++++++++++
 20 files changed, 148 insertions(+), 75 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 56fcd63..c6b1b40 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -352,7 +352,9 @@ your package adds with its own prefix, such as `acme_recipes`; unprefixed
 names belong to the built-ins, and
 [`architecture/plugins.md`](architecture/plugins.md) lists the names this
 covers. [`docs/extensions.md`](docs/extensions.md) builds a whole extension
-package, a probe with its bank, a scorer and their tests, step by step.
+package, a probe with its bank, a scorer and their tests, step by step. Import
+from `usersim.extension` and `usersim.testing`;
+[`docs/extension_api.md`](docs/extension_api.md) lists every name they export.
 
 To check your extension satisfies the contract:
 
diff --git a/architecture/plugins.md b/architecture/plugins.md
index 59ba9a8..9767f19 100644
--- a/architecture/plugins.md
+++ b/architecture/plugins.md
@@ -121,7 +121,7 @@ The scorer check is awaitable because a scorer is. The evaluator awaits every
 scorer, so `register_scorer` expects an async function:
 
 ```python
-from usersim.engine.evaluator.scorers import register_scorer
+from usersim.extension import register_scorer
 
 async def score(row, models):
     return {"my_axis": 1.0}
@@ -162,7 +162,7 @@ its package and names that `assets` directory on the probe class:
 ```python
 from pathlib import Path
 
-from usersim.engine.core.probes import BaseProbe
+from usersim.extension import BaseProbe
 
 class MyProbe(BaseProbe):
     label = "my_probe"
diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md
index 6ded75b..cbe6f16 100644
--- a/docs/engine/AUTHORING_A_PROBE.md
+++ b/docs/engine/AUTHORING_A_PROBE.md
@@ -223,7 +223,7 @@ the probe writes to `state.metadata`, and the label it is shown with. The panel
 leaves out a field whose value is empty, and is left out when every field is:
 
 ```python
-from usersim.engine.core.probes import PreviewField
+from usersim.extension import BankBackedProbe, PreviewField
 
 class MyProbe(BankBackedProbe):
     preview_fields = (
@@ -310,7 +310,7 @@ alias (not the class attr directly):
 
 ```python
 # In your generator.py
-from .bank import load_my_bank_default
+from usersim.engine.probes.my_probe.bank import load_my_bank_default
 
 _load_bank = load_my_bank_default  # tests patch this name
 
@@ -441,8 +441,7 @@ caught at import time, not at simulate-time when the locale's first
 row trips a KeyError.
 
 ```python
-from usersim.engine.core.locale import SHIPPED_LOCALES
-from usersim.engine.core.probes import LocalePromptPack
+from usersim.extension import SHIPPED_LOCALES, LocalePromptPack
 
 MY_USER_PROMPTS = LocalePromptPack(
     label="my_probe.user_system",
@@ -499,7 +498,7 @@ return prose and the gate will fail-retry until exhausted.
 ## Step 6: `@register_probe`
 
 ```python
-from usersim.engine.core.probes import register_probe
+from usersim.extension import register_probe
 
 
 @register_probe(
@@ -617,7 +616,7 @@ restriction) recipe-request templates. We'll use the
 **banked-verbatim** shape because the bank's questions are
 hand-authored and should reach the assistant unchanged.
 
-### File 1: `assets/cooking_recipes/sample.yaml`
+### File 1: `src/usersim/engine/assets/cooking_advisor/sample.yaml`
 
 ```yaml
 schema_version: "v0.1"
@@ -650,12 +649,11 @@ from __future__ import annotations
 
 import os
 from dataclasses import dataclass
+from pathlib import Path
 from typing import Optional
 
 import yaml
 
-from usersim.engine.core._assets import probe_assets_dir
-
 
 @dataclass
 class CookingRecipe:
@@ -682,10 +680,8 @@ class CookingBank:
 _CACHE: dict[str, CookingBank] = {}
 
 
-def load_cooking_bank_default() -> CookingBank:
-    path = os.environ.get("USERSIM_COOKING_BANK") or str(
-        probe_assets_dir("cooking_recipes") / "sample.yaml"
-    )
+def load_cooking_bank(probe_dir: Path) -> CookingBank:
+    path = os.environ.get("USERSIM_COOKING_BANK") or str(probe_dir / "sample.yaml")
     if path in _CACHE:
         return _CACHE[path]
     with open(path) as f:
@@ -724,7 +720,7 @@ def reset_cooking_bank_cache() -> None:
 ### File 4: `src/usersim/engine/probes/cooking_advisor/prompts.py`
 
 ```python
-from usersim.engine.core.probes import LocalePromptPack
+from usersim.extension import LocalePromptPack
 
 
 COOKING_USER_PROMPTS = LocalePromptPack(
@@ -761,25 +757,28 @@ from __future__ import annotations
 from typing import Any, Optional
 
 from usersim.engine.core.cooking_bank import (
-    CookingRecipe, load_cooking_bank_default,
-)
-from usersim.engine.core.outcomes import WarningKind
-from usersim.engine.core.probes import (
-    BankBackedProbe, BankVerbatimMixin, dispatch_episode, register_probe,
+    CookingRecipe, load_cooking_bank,
 )
+from usersim.engine.core.probes import dispatch_episode
 from usersim.engine.probes.cooking_advisor.prompts import (
     COOKING_GATE_PROMPT, COOKING_USER_PROMPTS,
 )
+from usersim.extension import (
+    BankBackedProbe, BankVerbatimMixin, OutcomeBuilder, Provenance,
+    WarningKind, register_probe,
+)
 
 PROBE_FAMILY = "cooking_advisor"
 PROBE_VARIANTS = ["default"]
 PROMPT_VERSION = "v1.0"
 
-_load_bank = load_cooking_bank_default
+_load_bank = load_cooking_bank
 
 
 def _bank_loader():
-    return _load_bank()
+    # The probe's own asset folder, <root>/cooking_advisor/, wherever
+    # the asset search path finds it.
+    return _load_bank(CookingAdvisorProbe.assets_dir())
 
 
 def should_succeed(state: Any) -> bool:
@@ -847,7 +846,6 @@ class CookingAdvisorProbe(BankVerbatimMixin, BankBackedProbe):
 async def simulate_cooking_advisor(
     models, data, persona, profile, locale, language, cfg, **kwargs,
 ):
-    from usersim.engine.core.outcomes import OutcomeBuilder, Provenance
     provenance = kwargs.get("provenance") or Provenance()
     outcome_builder = kwargs.get("outcome_builder") or OutcomeBuilder(
         provenance=provenance,
@@ -936,7 +934,7 @@ with a `ScriptedModel` answering for every model alias, so the test needs no
 provider and patches nothing:
 
 ```python
-from usersim.engine.config import ConversationSimulatorConfig
+from usersim.extension import ConversationSimulatorConfig
 from usersim.testing import run_episode
 
 
diff --git a/docs/extensions.md b/docs/extensions.md
index a10ee17..e58c7ce 100644
--- a/docs/extensions.md
+++ b/docs/extensions.md
@@ -96,8 +96,7 @@ from pathlib import Path
 
 import yaml
 
-from usersim.engine.core.identity import persona_uuid
-from usersim.engine.core.probes import BankBackedProbe, PreviewField, register_probe
+from usersim.extension import BankBackedProbe, PreviewField, persona_uuid, register_probe
 
 
 @dataclass(frozen=True)
@@ -201,7 +200,7 @@ recipes:
 ## The scorer
 
 ```python
-from usersim.engine.evaluator.scorers import register_scorer
+from usersim.extension import register_scorer
 
 
 async def score(row, models):
@@ -235,7 +234,7 @@ import json
 
 import acme_recipes.probe  # noqa: F401  registers the probe; discovery is off under test
 import acme_recipes.scorer  # noqa: F401  registers the scorer
-from usersim.engine.config import ConversationSimulatorConfig
+from usersim.extension import ConversationSimulatorConfig
 from usersim.testing import assert_hosted_parity, assert_probe_conforms, assert_scorer_conforms, run_episode
 
 PERSONA = {"first_name": "Sarah", "last_name": "Johnson", "age": 35, "city": "Seattle"}
diff --git a/templates/probe/README.md b/templates/probe/README.md
index 0467dfb..f004d6f 100644
--- a/templates/probe/README.md
+++ b/templates/probe/README.md
@@ -111,7 +111,7 @@ for the rationale.
 When porting one of the templates into a new module:
 
 - [ ] Pick a `label` for `@register_probe` that no other probe uses,
-      used by `--probe-mix` and the `_PROBE_REGISTRY` lookup.
+      used by `--probe-mix` and the probe registry.
       Underscores, lowercase.
 - [ ] Set `description`, a paragraph on what the probe tests; the
       per-row preview and the capability report show it.
diff --git a/templates/probe/generator_agentic.py b/templates/probe/generator_agentic.py
index c50cd63..5eef7ed 100644
--- a/templates/probe/generator_agentic.py
+++ b/templates/probe/generator_agentic.py
@@ -49,16 +49,16 @@
 from dataclasses import dataclass
 from typing import Any
 
-from usersim.engine.core.outcomes import (
-    OutcomeBuilder,
-    WarningKind,
-)
-from usersim.engine.core.probes import (
+from usersim.extension import (
     AgenticMixin,
     BankBackedProbe,
+    ConversationState,
+    OutcomeBuilder,
+    WarningKind,
+    finalize_completed,
+    make_result,
     register_probe,
 )
-from usersim.engine.core.simulation import ConversationState, finalize_completed, make_result
 
 
 @dataclass
diff --git a/templates/probe/generator_bank_reframing.py b/templates/probe/generator_bank_reframing.py
index 687812a..0fda1ff 100644
--- a/templates/probe/generator_bank_reframing.py
+++ b/templates/probe/generator_bank_reframing.py
@@ -61,10 +61,10 @@
 from dataclasses import dataclass
 from typing import Any
 
-from usersim.engine.core.outcomes import WarningKind
-from usersim.engine.core.probes import (
+from usersim.extension import (
     BankBackedProbe,
     BankReframingMixin,
+    WarningKind,
     register_probe,
 )
 
diff --git a/templates/probe/generator_bank_verbatim.py b/templates/probe/generator_bank_verbatim.py
index 6e49420..cd13026 100644
--- a/templates/probe/generator_bank_verbatim.py
+++ b/templates/probe/generator_bank_verbatim.py
@@ -49,17 +49,17 @@
 from pathlib import Path
 from typing import Any
 
-from usersim.engine.core.outcomes import WarningKind
-from usersim.engine.core.probes import (
+# After you copy this file into ``src/usersim/engine/probes/<your_probe>/``,
+# also copy ``prompts.py`` next to it and import from
+# ``usersim.engine.probes.<your_probe>.prompts``.
+from usersim.engine.templates.probe.prompts import DEMO_USER_AGENT_PROMPTS  # type: ignore[import-not-found]
+from usersim.extension import (
     BankBackedProbe,
     BankVerbatimMixin,
+    WarningKind,
     register_probe,
 )
 
-# After you copy this file into ``src/usersim/engine/probes/<your_probe>/``,
-# also copy ``prompts.py`` next to it and change this import to ``from .prompts``.
-from usersim.engine.templates.probe.prompts import DEMO_USER_AGENT_PROMPTS  # type: ignore[import-not-found]
-
 
 @dataclass
 class _DemoTask:
diff --git a/templates/probe/generator_baseprobe_only.py b/templates/probe/generator_baseprobe_only.py
index 996b002..9490061 100644
--- a/templates/probe/generator_baseprobe_only.py
+++ b/templates/probe/generator_baseprobe_only.py
@@ -25,15 +25,14 @@
 
 from __future__ import annotations
 
-from usersim.engine.core.probes import BaseProbe, register_probe
-
 # After you copy this file into ``src/usersim/engine/probes/<your_probe>/``,
-# also copy ``prompts.py`` next to it and change this import to a
-# relative one: ``from .prompts import ...``.
+# also copy ``prompts.py`` next to it and import from
+# ``usersim.engine.probes.<your_probe>.prompts``.
 from usersim.engine.templates.probe.prompts import (  # type: ignore[import-not-found]
     DEMO_GATE_PROMPT_TEMPLATE,
     DEMO_USER_AGENT_PROMPTS,
 )
+from usersim.extension import BaseProbe, register_probe
 
 
 @register_probe(
diff --git a/templates/probe/generator_custom_turn1.py b/templates/probe/generator_custom_turn1.py
index 570e930..c80f508 100644
--- a/templates/probe/generator_custom_turn1.py
+++ b/templates/probe/generator_custom_turn1.py
@@ -44,8 +44,7 @@
 from dataclasses import dataclass
 from typing import Any
 
-from usersim.engine.core.outcomes import WarningKind
-from usersim.engine.core.probes import BankBackedProbe, register_probe
+from usersim.extension import BankBackedProbe, WarningKind, register_probe
 
 
 @dataclass
diff --git a/templates/probe/generator_tool_calling.py b/templates/probe/generator_tool_calling.py
index 1af195d..8054b83 100644
--- a/templates/probe/generator_tool_calling.py
+++ b/templates/probe/generator_tool_calling.py
@@ -42,7 +42,7 @@
 import json
 from typing import Any
 
-from usersim.engine.core.probes import (
+from usersim.extension import (
     BaseProbe,
     ToolCallingMixin,
     register_probe,
diff --git a/templates/probe/prompts.py b/templates/probe/prompts.py
index 27ac6aa..dce3c6e 100644
--- a/templates/probe/prompts.py
+++ b/templates/probe/prompts.py
@@ -15,8 +15,7 @@
 
 from __future__ import annotations
 
-from usersim.engine.core.locale import SHIPPED_LOCALES
-from usersim.engine.core.probes import LocalePromptPack
+from usersim.extension import SHIPPED_LOCALES, LocalePromptPack
 
 # ── Multi-locale: use LocalePromptPack ──────────────────────────────
 
diff --git a/templates/probe/test_probe.py.template b/templates/probe/test_probe.py.template
index fb32a2c..c9b6442 100644
--- a/templates/probe/test_probe.py.template
+++ b/templates/probe/test_probe.py.template
@@ -57,12 +57,11 @@ from unittest.mock import patch
 import pytest
 
 import usersim.engine.generator  # noqa: F401 — bootstraps registry
-from usersim.engine.config import ConversationSimulatorConfig
-from usersim.engine.core.outcomes import (
-    FailureClass, OutcomeStatus, Provenance, WarningKind,
+from usersim.extension import (
+    ConversationSimulatorConfig, ConversationState, FailureClass,
+    OutcomeBuilder, OutcomeStatus, Provenance, WarningKind,
 )
-from usersim.engine.core.probes import _PROBE_REGISTRY, resolve_probe
-from usersim.testing import run_episode
+from usersim.testing import assert_probe_conforms, run_episode
 
 # Replace with your probe's module + names.
 from usersim.engine.probes.YOUR_PROBE import (
@@ -188,13 +187,11 @@ class TestSimulateYourProbe:
         )
         assert run.row["conversation_status"] is True
 
-    # 1. Registry registration.
-    def test_registered_in_probe_registry(self) -> None:
-        assert PROBE_LABEL in _PROBE_REGISTRY
-        probe_cls = resolve_probe(PROBE_LABEL)
-        assert probe_cls.family == PROBE_LABEL
-        assert probe_cls.prompt_version  # non-empty
-        assert probe_cls.variants  # non-empty
+    # 1. Registered, with the family, prompt version and variants
+    #    @register_probe sets, and conforming to the probe contract.
+    def test_registered_and_conforms(self) -> None:
+        assert_probe_conforms(PROBE_LABEL)
+        assert probe_gen.YourProbe.family == PROBE_LABEL  # rename to your class
 
     # 2. Single-turn end-to-end (verbatim or generate-and-gate).
     async def test_single_turn_end_to_end(
@@ -282,15 +279,11 @@ class TestSimulateYourProbe:
 
 class TestShouldSucceed:
     def test_returns_true_when_required_metadata_set(self) -> None:
-        from usersim.engine.core.outcomes import OutcomeBuilder
-        from usersim.engine.core.simulation import ConversationState
         state = ConversationState(outcome=OutcomeBuilder())
         # Set the probe's join key.
         state.metadata["YOUR_REQUIRED_KEY"] = "id-001"
         assert probe_gen.should_succeed(state) is True
 
     def test_returns_false_when_required_metadata_missing(self) -> None:
-        from usersim.engine.core.outcomes import OutcomeBuilder
-        from usersim.engine.core.simulation import ConversationState
         state = ConversationState(outcome=OutcomeBuilder())
         assert probe_gen.should_succeed(state) is False
diff --git a/tests/fixtures/extension_package/usersim_extension_fixture/backend.py b/tests/fixtures/extension_package/usersim_extension_fixture/backend.py
index 56c5d98..2e83aec 100644
--- a/tests/fixtures/extension_package/usersim_extension_fixture/backend.py
+++ b/tests/fixtures/extension_package/usersim_extension_fixture/backend.py
@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from usersim.engine.core.backends import JobHandle, JobStatus
+from usersim.extension import JobHandle, JobStatus
 
 
 class FixtureBackend:
diff --git a/tests/fixtures/extension_package/usersim_extension_fixture/capability.py b/tests/fixtures/extension_package/usersim_extension_fixture/capability.py
index c88b15d..56057f6 100644
--- a/tests/fixtures/extension_package/usersim_extension_fixture/capability.py
+++ b/tests/fixtures/extension_package/usersim_extension_fixture/capability.py
@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from usersim.taxonomy.capabilities import CapabilityDefinition, EvidenceSource
+from usersim.extension import CapabilityDefinition, EvidenceSource
 
 CAPABILITY = CapabilityDefinition(
     id="fixture_capability",
diff --git a/tests/fixtures/extension_package/usersim_extension_fixture/domain.py b/tests/fixtures/extension_package/usersim_extension_fixture/domain.py
index 8421c8c..39a3df0 100644
--- a/tests/fixtures/extension_package/usersim_extension_fixture/domain.py
+++ b/tests/fixtures/extension_package/usersim_extension_fixture/domain.py
@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from usersim.asset_gen.registry import AssetDomain
+from usersim.extension import AssetDomain
 
 
 def _noop(*args, **kwargs):
diff --git a/tests/fixtures/extension_package/usersim_extension_fixture/identity.py b/tests/fixtures/extension_package/usersim_extension_fixture/identity.py
index 63420b3..a7d2d6f 100644
--- a/tests/fixtures/extension_package/usersim_extension_fixture/identity.py
+++ b/tests/fixtures/extension_package/usersim_extension_fixture/identity.py
@@ -3,8 +3,8 @@
 
 from pathlib import Path
 
-from usersim.engine.core.probes import register_probe
 from usersim.engine.probes.identity_disclosure.generator import IdentityDisclosureProbe
+from usersim.extension import register_probe
 
 
 @register_probe(family="fixture_identity", prompt_version="v1.0", variants=("default",))
diff --git a/tests/fixtures/extension_package/usersim_extension_fixture/probe.py b/tests/fixtures/extension_package/usersim_extension_fixture/probe.py
index b3ad86d..e4f472a 100644
--- a/tests/fixtures/extension_package/usersim_extension_fixture/probe.py
+++ b/tests/fixtures/extension_package/usersim_extension_fixture/probe.py
@@ -5,7 +5,7 @@
 
 import yaml
 
-from usersim.engine.core.probes import BaseProbe, PreviewField, register_probe
+from usersim.extension import BaseProbe, PreviewField, register_probe
 
 
 def _check_locale_bank(locale_dir: Path) -> None:
diff --git a/tests/fixtures/extension_package/usersim_extension_fixture/scorer.py b/tests/fixtures/extension_package/usersim_extension_fixture/scorer.py
index 93f6512..37d4611 100644
--- a/tests/fixtures/extension_package/usersim_extension_fixture/scorer.py
+++ b/tests/fixtures/extension_package/usersim_extension_fixture/scorer.py
@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from usersim.engine.evaluator.scorers import register_scorer
+from usersim.extension import register_scorer
 
 
 async def score(row, models):
diff --git a/tests/test_extension_api.py b/tests/test_extension_api.py
index 653663e..64e35a5 100644
--- a/tests/test_extension_api.py
+++ b/tests/test_extension_api.py
@@ -8,12 +8,18 @@
 imports each name on first use, so the tests also check that every name
 resolves to the object its home module defines and that the imports a type
 checker reads name the same objects.
+
+The code an extension author copies (the probe templates, the extension
+fixture, and the Python examples in the extension docs) imports ``usersim``
+only through these two modules, apart from the imports listed in
+``_KEPT_INTERNAL_IMPORTS``.
 """
 
 from __future__ import annotations
 
 import ast
 import importlib
+import re
 import subprocess
 import sys
 from pathlib import Path
@@ -26,6 +32,43 @@
 _REPO = Path(__file__).resolve().parents[1]
 _REFERENCE = _REPO / "docs" / "extension_api.md"
 
+_PUBLIC = {module.__name__: set(module.__all__) for module in (usersim.extension, usersim.testing)}
+
+#: Files an extension author copies code from.
+_EXAMPLE_FILES = (
+    *sorted((_REPO / "templates" / "probe").glob("*.py")),
+    _REPO / "templates" / "probe" / "test_probe.py.template",
+    *sorted((_REPO / "tests" / "fixtures" / "extension_package" / "usersim_extension_fixture").glob("*.py")),
+)
+
+#: Pages whose Python examples an extension author copies.
+_EXAMPLE_DOCS = (
+    "CONTRIBUTING.md",
+    "architecture/plugins.md",
+    "docs/engine/AUTHORING_A_PROBE.md",
+    "docs/extension_api.md",
+    "docs/extensions.md",
+)
+
+#: Imports from other ``usersim`` modules that the examples keep, as (module, name).
+_KEPT_INTERNAL_IMPORTS = {
+    # A new in-tree probe's own modules, under the names the examples give them.
+    ("usersim.engine.core.cooking_bank", "CookingRecipe"),
+    ("usersim.engine.core.cooking_bank", "load_cooking_bank"),
+    ("usersim.engine.probes.YOUR_PROBE", "generator"),
+    ("usersim.engine.probes.cooking_advisor.prompts", "COOKING_GATE_PROMPT"),
+    ("usersim.engine.probes.cooking_advisor.prompts", "COOKING_USER_PROMPTS"),
+    ("usersim.engine.probes.my_probe.bank", "load_my_bank_default"),
+    ("usersim.engine.templates.probe.prompts", "DEMO_GATE_PROMPT_TEMPLATE"),
+    ("usersim.engine.templates.probe.prompts", "DEMO_USER_AGENT_PROMPTS"),
+    # The worked example's in-tree shim, and the test template's standard
+    # tests, which load the probe registry before patching the loop's model calls.
+    ("usersim.engine.core.probes", "dispatch_episode"),
+    ("usersim.engine.generator", None),
+    # The fixture's probe that layers a spec over a shipped probe's.
+    ("usersim.engine.probes.identity_disclosure.generator", "IdentityDisclosureProbe"),
+}
+
 
 def _documented(module: str) -> list[str]:
     """The names in the first column of the tables under ``## `module```."""
@@ -87,3 +130,44 @@ def test_an_asset_domain_or_backend_import_leaves_the_conversation_loop_unloaded
     result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
     assert result.returncode == 0, result.stderr
     assert result.stdout.strip() == "[]", f"loaded by the import: {result.stdout.strip()}"
+
+
+def _examples() -> list[tuple[str, str]]:
+    """Each example as (where, source): a whole file, or one Python block of a page."""
+    examples = [(path.relative_to(_REPO).as_posix(), path.read_text(encoding="utf-8")) for path in _EXAMPLE_FILES]
+    for doc in _EXAMPLE_DOCS:
+        blocks = re.findall(r"```python\n(.*?)```", (_REPO / doc).read_text(encoding="utf-8"), re.S)
+        examples += [(f"{doc}, Python block {index + 1}", block) for index, block in enumerate(blocks)]
+    return examples
+
+
+def _usersim_imports(source: str) -> list[tuple[str, str | None]]:
+    """Every ``usersim`` import in ``source`` as (module, name); name is None for ``import module``."""
+    imports: list[tuple[str, str | None]] = []
+    for node in ast.walk(ast.parse(source)):
+        if isinstance(node, ast.ImportFrom) and (node.module or "").split(".")[0] == "usersim":
+            imports += [(node.module, alias.name) for alias in node.names]
+        elif isinstance(node, ast.Import):
+            imports += [(alias.name, None) for alias in node.names if alias.name.split(".")[0] == "usersim"]
+    return imports
+
+
+def test_the_examples_import_usersim_through_the_public_modules():
+    problems, kept = [], set()
+    for where, source in _examples():
+        try:
+            imports = _usersim_imports(source)
+        except SyntaxError:
+            # A pseudo-code block cannot be checked, so it may not import usersim at all.
+            if re.search(r"^\s*(from|import) usersim", source, re.M):
+                problems.append(f"{where}: does not parse, and imports usersim")
+            continue
+        for module, name in imports:
+            if name in _PUBLIC.get(module, ()):
+                continue
+            if (module, name) in _KEPT_INTERNAL_IMPORTS:
+                kept.add((module, name))
+            else:
+                problems.append(f"{where}: {module} {name or ''}".rstrip())
+    assert not problems, "import these from usersim.extension or usersim.testing:\n" + "\n".join(problems)
+    assert kept == _KEPT_INTERNAL_IMPORTS, f"no example makes these imports: {sorted(_KEPT_INTERNAL_IMPORTS - kept)}"

From 57ed9b8e8e15e7e9230f09adb5b0d345ccf236a7 Mon Sep 17 00:00:00 2001
From: Yev Meyer <ymeyer@nvidia.com>
Date: Sun, 1 Nov 2026 14:59:41 -0500
Subject: [PATCH 5/6] docs(extension): replace the examples with a multilingual
 booking probe

docs/extensions.md builds an extension package, my_bookings, and the authoring guide's worked example builds the same probe in tree, as bookings. A simulated user asks an assistant that has a book_appointment(date, time, people) tool to book a tour. The en_US and pt_BR banks both open with 04/03/2027, which is 3 April to the first user and 4 March to the second, and the user gives the number of people only when the assistant asks.

The probe answers the tool call through ToolExecutionMixin, records the first booking, and lets a conversation end only once the assistant has booked. The extension's scorer checks each argument against what the user meant and whether the assistant asked before booking, and its tests run an episode in pt_BR offline. The in-tree version adds what a built-in probe needs: a LocalePromptPack, a WarningKind member, the BUILTIN_PROBE_MODULES entry and an asset_problems check.

Signed-off-by: Yev Meyer <ymeyer@nvidia.com>
---
 CONTRIBUTING.md                  |   2 +-
 architecture/plugins.md          |   5 +-
 docs/engine/AUTHORING_A_PROBE.md | 395 +++++++++++++++++--------------
 docs/extensions.md               | 353 +++++++++++++++++++--------
 tests/test_extension_api.py      |   8 +-
 5 files changed, 484 insertions(+), 279 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c6b1b40..8d69673 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -348,7 +348,7 @@ through entry points; see the seven groups in
 `src/usersim/engine/core/_extensions.py`. Give a probe or scorer a name no
 other probe or scorer uses: registering one under a taken name raises an
 error naming both, and an extension that does so is skipped. Start every name
-your package adds with its own prefix, such as `acme_recipes`; unprefixed
+your package adds with its own prefix, such as `my_bookings`; unprefixed
 names belong to the built-ins, and
 [`architecture/plugins.md`](architecture/plugins.md) lists the names this
 covers. [`docs/extensions.md`](docs/extensions.md) builds a whole extension
diff --git a/architecture/plugins.md b/architecture/plugins.md
index 9767f19..705fb2f 100644
--- a/architecture/plugins.md
+++ b/architecture/plugins.md
@@ -36,13 +36,14 @@ my_probe = "my_package.probe:MyProbe"
 **Every name an extension adds carries its prefix.** Probe labels, scorer
 names, capability ids, asset domains, command names, output columns, warning
 kinds and metadata keys all start with the package's own prefix, such as
-`acme_recipes` or `acme_used_placeholder_recipe`. Unprefixed names belong to
+`my_bookings` or `my_used_placeholder_booking`. Unprefixed names belong to
 the built-ins. The registries refuse a name that is already taken, so the
 prefix is what keeps two packages, or a package and a later built-in, from
 claiming the same one.
 
 [`docs/extensions.md`](../docs/extensions.md) builds a whole extension
-package: a probe with its own bank, a scorer, their tests and a CI recipe.
+package: a probe with a bank in each of two languages, a scorer, their tests
+and a CI recipe.
 [`docs/extension_api.md`](../docs/extension_api.md) lists every name an
 extension imports, from `usersim.extension` and `usersim.testing`.
 
diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md
index cbe6f16..08b3c39 100644
--- a/docs/engine/AUTHORING_A_PROBE.md
+++ b/docs/engine/AUTHORING_A_PROBE.md
@@ -24,7 +24,7 @@ piece to pick and *how* to wire it.
 6. [Step 5: write the user-judge gate prompt](#step-5-write-the-user-judge-gate-prompt)
 7. [Step 6: `@register_probe`](#step-6-register_probe)
 8. [Step 7: run the smoke test](#step-7-run-the-smoke-test)
-9. [Worked example: a `cooking_advisor` probe end-to-end](#worked-example-a-cooking_advisor-probe-end-to-end)
+9. [Worked example: a `bookings` probe end-to-end](#worked-example-a-bookings-probe-end-to-end)
 10. [Common gotchas](#common-gotchas)
 11. [Test authoring](#test-authoring)
 12. [Wiring the out-of-sim scorer](#wiring-the-out-of-sim-scorer)
@@ -199,7 +199,7 @@ placeholder content, and the capability report counts those rows per cell. A
 probe in this repository uses a `WarningKind.USED_PLACEHOLDER_*` member. A
 probe shipped in its own package names a kind of its own, as a string of the
 form `<prefix>_used_placeholder_<thing>`, such as
-`"acme_used_placeholder_recipe"`; the report counts it like the built-in kinds.
+`"my_used_placeholder_booking"`; the report counts it like the built-in kinds.
 `@register_probe` refuses a kind that does not mark placeholder content. Other
 warnings a probe adds with `add_warning` take the same prefixed form, such as
 `"acme_slow_tool"`.
@@ -456,7 +456,7 @@ MY_USER_PROMPTS = LocalePromptPack(
 
 
 # At simulate time:
-prompt = MY_USER_PROMPTS.get(self._locale, persona="Sarah", topic="cooking")
+prompt = MY_USER_PROMPTS.get(self._locale, persona="Sarah", topic="train tickets")
 ```
 
 If you only ship one locale, plain string templates are fine:
@@ -608,181 +608,180 @@ import error or a name another package already uses.
 
 ---
 
-## Worked example: a `cooking_advisor` probe end-to-end
+## Worked example: a `bookings` probe end-to-end
 
-Let's build a complete probe: `cooking_advisor` asks the assistant
-for cooking help, with a curated bank of (cuisine × dietary
-restriction) recipe-request templates. We'll use the
-**banked-verbatim** shape because the bank's questions are
-hand-authored and should reach the assistant unchanged.
+Let's build a complete probe. `bookings` checks that an assistant turns what a
+user writes into the right tool call. The user asks it to book an appointment,
+giving the date and time the way people write them in the user's language, and
+the assistant has a `book_appointment` tool that takes them in one fixed form.
+The probe ships banks for `en_US` and `pt_BR`; the same probe, shipped in its
+own package, is the example in [`docs/extensions.md`](../extensions.md).
 
-### File 1: `src/usersim/engine/assets/cooking_advisor/sample.yaml`
+We'll use the **banked-verbatim** shape, so each date reaches the assistant
+exactly as the bank writes it, with `ToolExecutionMixin` to answer the
+assistant's tool call.
+
+### File 1: the banks, `src/usersim/engine/assets/bookings/<locale>/bookings.yaml`
+
+`en_US/bookings.yaml`:
 
 ```yaml
-schema_version: "v0.1"
-bank_id: "cooking_recipes_sample"
-bank_version: "v0.1.0"
-placeholder: true
-recipes:
-  - id: "REC-001"
+bank_id: bookings_en_US
+bank_version: "1.0.0"
+bookings:
+  - id: museum_tour
+    opening: Hi! Can you book the guided museum tour for 04/03/2027 at 3:30 pm?
+    date: "2027-04-03"
+    time: "15:30"
+    people: 3
     placeholder: true
-    cuisine: "italian"
-    dietary: "vegetarian"
-    persona_tags: ["interest:cooking"]
-    question: "I have eggplant, mozzarella, and tomato. Can you walk me through making eggplant parmigiana for 4 people?"
-    expected_themes: ["layering", "salting eggplant", "baking time", "cheese melt"]
-  - id: "REC-002"
+```
+
+`pt_BR/bookings.yaml`:
+
+```yaml
+bank_id: bookings_pt_BR
+bank_version: "1.0.0"
+bookings:
+  - id: museum_tour
+    opening: Oi! Você pode reservar a visita guiada ao museu para 04/03/2027 às 15h30?
+    date: "2027-03-04"
+    time: "15:30"
+    people: 3
     placeholder: true
-    cuisine: "japanese"
-    dietary: "gluten_free"
-    persona_tags: ["interest:cooking"]
-    question: "My partner has celiac. What's a gluten-free version of teriyaki sauce that still tastes right?"
-    expected_themes: ["tamari substitution", "rice vinegar", "mirin alternatives"]
 ```
 
-### File 2: `src/usersim/engine/core/cooking_bank.py`
+Both open with `04/03/2027`: 3 April to a user writing American English, and
+4 March to one writing Brazilian Portuguese. Neither says how many people are
+coming, so an assistant that books on its first reply has to guess.
+
+### File 2: `src/usersim/engine/probes/bookings/bank.py`
 
 ```python
-"""Cooking-recipe bank loader."""
+"""Booking banks: one per locale, under the probe's asset folder."""
 
 from __future__ import annotations
 
-import os
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional
 
 import yaml
 
+#: The locales the probe ships a bank for.
+LOCALES = ("en_US", "pt_BR")
+
 
-@dataclass
-class CookingRecipe:
+@dataclass(frozen=True)
+class Booking:
     id: str
+    opening: str  # turn 1, in the locale's language; leaves out how many people
+    date: str  # what the user means, as the tool takes it: YYYY-MM-DD
+    time: str  # HH:MM, 24-hour
+    people: int  # given only when the assistant asks
     placeholder: bool
-    cuisine: str
-    dietary: str
-    persona_tags: tuple[str, ...]
-    question: str
-    expected_themes: tuple[str, ...]
 
 
-@dataclass
-class CookingBank:
+@dataclass(frozen=True)
+class BookingBank:
     bank_id: str
     bank_version: str
-    placeholder: bool
-    recipes: tuple[CookingRecipe, ...]
-
-    def by_id(self, recipe_id: str) -> Optional[CookingRecipe]:
-        return next((r for r in self.recipes if r.id == recipe_id), None)
-
-
-_CACHE: dict[str, CookingBank] = {}
-
-
-def load_cooking_bank(probe_dir: Path) -> CookingBank:
-    path = os.environ.get("USERSIM_COOKING_BANK") or str(probe_dir / "sample.yaml")
-    if path in _CACHE:
-        return _CACHE[path]
-    with open(path) as f:
-        data = yaml.safe_load(f)
-    bank = CookingBank(
-        bank_id=data["bank_id"],
-        bank_version=data["bank_version"],
-        placeholder=data.get("placeholder", False),
-        recipes=tuple(
-            CookingRecipe(
-                id=r["id"],
-                placeholder=r.get("placeholder", False),
-                cuisine=r["cuisine"],
-                dietary=r["dietary"],
-                persona_tags=tuple(r.get("persona_tags", [])),
-                question=r["question"],
-                expected_themes=tuple(r.get("expected_themes", [])),
-            )
-            for r in data["recipes"]
-        ),
-    )
-    _CACHE[path] = bank
-    return bank
+    bookings: tuple[Booking, ...]
 
 
-def reset_cooking_bank_cache() -> None:
-    _CACHE.clear()
+def read_booking_bank(locale_dir: Path) -> BookingBank:
+    doc = yaml.safe_load((locale_dir / "bookings.yaml").read_text(encoding="utf-8"))
+    return BookingBank(doc["bank_id"], doc["bank_version"], tuple(Booking(**b) for b in doc["bookings"]))
 ```
 
-### File 3: `src/usersim/engine/probes/cooking_advisor/__init__.py`
+### File 3: `src/usersim/engine/probes/bookings/__init__.py`
 
 ```python
-"""cooking_advisor probe: banked-verbatim recipe questions."""
+"""bookings probe: booking arguments from dates and times as users write them."""
 ```
 
-### File 4: `src/usersim/engine/probes/cooking_advisor/prompts.py`
+### File 4: `src/usersim/engine/probes/bookings/prompts.py`
 
 ```python
-from usersim.extension import LocalePromptPack
+"""The simulated user's instructions, in each language the probe ships."""
 
+from usersim.engine.probes.bookings.bank import LOCALES
+from usersim.extension import LocalePromptPack
 
-COOKING_USER_PROMPTS = LocalePromptPack(
-    label="cooking_advisor.user_system",
+BOOKINGS_USER_PROMPTS = LocalePromptPack(
+    label="bookings.user_system",
     prompts={
         "en_US": (
-            "You are {persona}, a home cook in the US who needs help with "
-            "a recipe. Stay in your voice; do not break the fourth wall. "
-            "Ask follow-up questions naturally if the response is unclear."
+            "{persona}\n\n"
+            "You asked an assistant to book an appointment, giving the date and time. "
+            "If it asks how many people are coming, the answer is {people}; say it the way you would. "
+            "Once it confirms the booking, thank it."
+        ),
+        "pt_BR": (
+            "{persona}\n\n"
+            "Você pediu a um assistente para marcar um horário e já disse a data e a hora. "
+            "Se ele perguntar quantas pessoas vão, a resposta é {people}; diga isso do seu jeito. "
+            "Quando ele confirmar a reserva, agradeça."
         ),
-        # Add more locales as you ship them.
     },
-    shipped_locales=("en_US",),
-)
-
-
-COOKING_GATE_PROMPT = (
-    "You are evaluating whether a user message is a legitimate cooking "
-    "question (PASS) or a role-violation / off-topic (FAIL).\n\n"
-    "Conversation so far:\n{conversation_history}\n\n"
-    "User message:\n{user_turn_to_evaluate}\n\n"
-    "Respond with <rating>PASS</rating> or <rating>FAIL</rating> on the "
-    "first line, then a brief <explanation>...</explanation>."
+    shipped_locales=LOCALES,
 )
 ```
 
-### File 5: `src/usersim/engine/probes/cooking_advisor/generator.py`
+The user's instructions are written in each language the probe ships, and
+`LocalePromptPack` refuses, when the module is imported, a locale without them.
+
+### File 5: `src/usersim/engine/probes/bookings/generator.py`
 
 ```python
-"""cooking_advisor probe: banked-verbatim recipe questions."""
+"""bookings probe: booking arguments from dates and times as users write them."""
 
 from __future__ import annotations
 
-from typing import Any, Optional
+import json
+import random
+from typing import Any
 
-from usersim.engine.core.cooking_bank import (
-    CookingRecipe, load_cooking_bank,
-)
 from usersim.engine.core.probes import dispatch_episode
-from usersim.engine.probes.cooking_advisor.prompts import (
-    COOKING_GATE_PROMPT, COOKING_USER_PROMPTS,
-)
+from usersim.engine.probes.bookings.bank import LOCALES, Booking, read_booking_bank
+from usersim.engine.probes.bookings.prompts import BOOKINGS_USER_PROMPTS
 from usersim.extension import (
-    BankBackedProbe, BankVerbatimMixin, OutcomeBuilder, Provenance,
-    WarningKind, register_probe,
+    BankBackedProbe,
+    BankVerbatimMixin,
+    OutcomeBuilder,
+    Provenance,
+    ToolExecutionMixin,
+    WarningKind,
+    format_persona_for_prompt,
+    persona_uuid,
+    register_probe,
 )
 
-PROBE_FAMILY = "cooking_advisor"
+PROBE_FAMILY = "bookings"
 PROBE_VARIANTS = ["default"]
 PROMPT_VERSION = "v1.0"
 
-_load_bank = load_cooking_bank
-
+BOOK_APPOINTMENT = {
+    "type": "function",
+    "function": {
+        "name": "book_appointment",
+        "description": "Book an appointment.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "date": {"type": "string", "description": "YYYY-MM-DD"},
+                "time": {"type": "string", "description": "HH:MM, 24-hour"},
+                "people": {"type": "integer"},
+            },
+            "required": ["date", "time", "people"],
+        },
+    },
+}
 
-def _bank_loader():
-    # The probe's own asset folder, <root>/cooking_advisor/, wherever
-    # the asset search path finds it.
-    return _load_bank(CookingAdvisorProbe.assets_dir())
+_load_bank = read_booking_bank
 
 
-def should_succeed(state: Any) -> bool:
-    return bool(state.metadata.get("recipe_id"))
+def _bank_loader(locale: str):
+    return _load_bank(BookingsProbe.assets_dir() / locale)
 
 
 @register_probe(
@@ -790,69 +789,91 @@ def should_succeed(state: Any) -> bool:
     prompt_version=PROMPT_VERSION,
     variants=tuple(PROBE_VARIANTS),
 )
-class CookingAdvisorProbe(BankVerbatimMixin, BankBackedProbe):
-    label = "cooking_advisor"
+class BookingsProbe(ToolExecutionMixin, BankVerbatimMixin, BankBackedProbe):
+    label = "bookings"
+    description = (
+        "Tests whether an assistant books what the user means: a date, a time and a number of people, "
+        "written the way people write them in the user's language."
+    )
     bank_loader = staticmethod(_bank_loader)
-    placeholder_warning_kind = WarningKind.USED_PLACEHOLDER_FACT
-    bank_version_key = "cooking"  # locale-independent, this probe is en-only
-
-    def derive_task(
-        self, persona: dict, bank: Any, *, cfg: Any,
-    ) -> Optional[CookingRecipe]:
-        # Narrow by persona tags as the shipped probes do; this example
-        # takes the first match.
-        if not bank.recipes:
-            return None
-        return bank.recipes[0]
+    locale_bank_check = staticmethod(read_booking_bank)
+    placeholder_warning_kind = WarningKind.USED_PLACEHOLDER_BOOKING
+    verbatim_field = "opening"
+    output_columns = ("booking_id", "booking")
+
+    @classmethod
+    def asset_problems(cls) -> list[str]:
+        problems = []
+        for locale in LOCALES:
+            try:
+                read_booking_bank(cls.assets_dir() / locale)
+            except Exception as error:
+                problems.append(f"{locale}: {type(error).__name__}: {error}")
+        return problems
+
+    def derive_task(self, persona: dict, bank: Any, *, cfg: Any) -> Booking:
+        rng = random.Random(f"{persona_uuid(persona)}:{bank.bank_version}:{cfg.random_seed}")
+        return rng.choice(bank.bookings)
 
     def get_user_system_prompt(self) -> str:
-        return COOKING_USER_PROMPTS.get(
-            self._locale, persona=self._persona.get("first_name", "the user"),
+        return BOOKINGS_USER_PROMPTS.get(
+            self._locale,
+            persona=format_persona_for_prompt(self._persona),
+            people=self._task.people,
         )
 
     def get_assistant_system_prompt(self) -> str:
         return ""  # Pure-capability-test policy.
 
-    def format_gate_prompt(
-        self, user_query: str, conversation_history: str,
-    ) -> str:
-        return COOKING_GATE_PROMPT.format(
-            user_turn_to_evaluate=user_query,
-            conversation_history=conversation_history,
-        )
+    def get_tools_for_assistant(self) -> list:
+        return [BOOK_APPOINTMENT]
 
-    async def get_verbatim_first_user_turn(self, state: Any) -> str:
-        state.metadata["recipe_id"] = self._task.id
-        state.metadata["cuisine"] = self._task.cuisine
-        state.metadata["dietary"] = self._task.dietary
-        state.metadata["bank_id"] = self._bank.bank_id
-        state.metadata["bank_version"] = self._bank.bank_version
-        return self._task.question
+    def allow_early_stop_at_turn(self, turn_idx: int, state: Any) -> bool:
+        return "booking" in state.metadata
 
-    def should_succeed(self, state: Any) -> bool:
-        return should_succeed(state)
+    async def execute_tool_call(self, name, args, tc, state, models, *, turn_idx, call_idx) -> str:
+        if "booking" not in state.metadata:
+            user_turns = sum(1 for message in state.messages if message.get("role") == "user")
+            state.metadata["booking"] = {"arguments": args, "after_user_turns": user_turns}
+        return json.dumps({"status": "booked"})
 
     def build_result_extras(self, state: Any) -> dict:
-        extras = super().build_result_extras(state)
-        if self._task is not None:
-            extras["probe_variant"] = f"{self._task.cuisine}::{self._task.dietary}"
-            extras["recipe_id"] = self._task.id
-            extras["cuisine"] = self._task.cuisine
-            extras["dietary"] = self._task.dietary
-        return extras
-
-
-# Optional thin shim for callers that import simulate_cooking_advisor.
-async def simulate_cooking_advisor(
-    models, data, persona, profile, locale, language, cfg, **kwargs,
+        task = self._task
+        booking = {
+            "expected": {"date": task.date, "time": task.time, "people": task.people},
+            "made": state.metadata.get("booking"),
+        }
+        return {
+            **super().build_result_extras(state),
+            "booking_id": task.id,
+            "booking": json.dumps(booking, ensure_ascii=False),
+        }
+
+
+# Optional thin shim for callers that import simulate_bookings.
+async def simulate_bookings(
+    models,
+    data,
+    persona,
+    profile,
+    locale,
+    language,
+    cfg,
+    **kwargs,
 ):
     provenance = kwargs.get("provenance") or Provenance()
     outcome_builder = kwargs.get("outcome_builder") or OutcomeBuilder(
         provenance=provenance,
     )
-    probe = CookingAdvisorProbe(
-        persona=persona, locale=locale, language=language, models=models,
-        cfg=cfg, provenance=provenance, profile=profile, data=data,
+    probe = BookingsProbe(
+        persona=persona,
+        locale=locale,
+        language=language,
+        models=models,
+        cfg=cfg,
+        provenance=provenance,
+        profile=profile,
+        data=data,
         outcome_builder=outcome_builder,
     )
     # dispatch_episode installs the run's per-episode settings (reasoning
@@ -860,7 +881,30 @@ async def simulate_cooking_advisor(
     return await dispatch_episode(probe, models=models, data=data, cfg=cfg)
 ```
 
-### File 6: built-in probe list
+- `execute_tool_call` records the first booking in `state.metadata` during the
+  run, with how many user turns came before it, and `build_result_extras`
+  writes it to the `booking` column beside what the user meant. A scorer
+  compares the two, as [the extension guide's scorer](../extensions.md#the-scorer)
+  does.
+- `allow_early_stop_at_turn` lets the conversation end only once the
+  assistant has booked, so a user who thanks it early does not end it first.
+- `should_succeed` keeps its default: an assistant that never books has not
+  broken the probe's protocol, so its row is scored like any other.
+- The probe keeps `BaseProbe`'s gate prompt (Step 5).
+
+### File 6: the placeholder warning
+
+In [`src/usersim/engine/core/outcomes.py`](../../src/usersim/engine/core/outcomes.py),
+add the probe's kind to `WarningKind`, beside the other `USED_PLACEHOLDER_*`
+members:
+
+```python
+    # Flagged when a bookings trajectory ran against a booking marked
+    # ``placeholder: true`` in its bank.
+    USED_PLACEHOLDER_BOOKING = "used_placeholder_booking"
+```
+
+### File 7: built-in probe list
 
 In [`src/usersim/engine/core/probes.py`](../../src/usersim/engine/core/probes.py),
 add the module to `BUILTIN_PROBE_MODULES`:
@@ -868,7 +912,7 @@ add the module to `BUILTIN_PROBE_MODULES`:
 ```python
 BUILTIN_PROBE_MODULES: tuple[str, ...] = (
     ...
-    "usersim.engine.probes.cooking_advisor.generator",
+    "usersim.engine.probes.bookings.generator",
 )
 ```
 
@@ -889,23 +933,24 @@ names and arguments; `ProbeEpisodeRuntime` assigns the indices. Author probe
 logic against those supplied indices rather than reconstructing host
 transcript positions.
 
-### File 7: tests at `tests/engine/probes/test_cooking_advisor.py`
+### File 8: tests at `tests/engine/probes/test_bookings.py`
 
 Copy [`templates/probe/test_probe.py.template`](../../templates/probe/test_probe.py.template),
-substitute `cooking_advisor` for `YOUR_PROBE`, fill in the 8 standard
+substitute `bookings` for `YOUR_PROBE`, fill in the 8 standard
 test stubs.
 
 ### Run it
 
 ```bash
-usersim smoke                                                    # confirms registration
-pytest tests/engine/probes/test_cooking_advisor.py -v # confirms behavior
-usersim simulate --locale en_US --num-rows 5 \
-  --probe-mix 'cooking_advisor=1.0' --out output/cooking   # end-to-end
+usersim smoke                                      # confirms registration
+pytest tests/engine/probes/test_bookings.py -v     # confirms behavior
+usersim simulate --locale pt_BR --num-rows 5 \
+  --probe-mix 'bookings=1.0' --out output/bookings # end-to-end
 ```
 
-That is the whole probe: a bank loader, prompts, the probe class and the
-shim. Everything about how the conversation runs comes from the shared loop.
+That is the whole probe: two banks and their loader, the user's prompts, the
+probe class with its tool, and the shim. Everything about how the conversation
+runs comes from the shared loop.
 
 ---
 
@@ -944,7 +989,7 @@ async def test_my_probe_runs_end_to_end():
         config=ConversationSimulatorConfig(name="conversation_messages", max_turns=2),
         persona={"first_name": "Sarah", "last_name": "Johnson", "age": 35, "city": "Seattle",
                  "education_level": "Bachelor", "occupation": "Software Engineer"},
-        data={"theme": {"type": "cooking", "description": "Planning weeknight dinners."}},
+        data={"theme": {"type": "travel", "description": "Planning a weekend away."}},
     )
     assert run.row["conversation_status"] is True
 ```
diff --git a/docs/extensions.md b/docs/extensions.md
index e58c7ce..886ece4 100644
--- a/docs/extensions.md
+++ b/docs/extensions.md
@@ -2,9 +2,10 @@
 
 A probe, a scorer or another addition can ship in its own Python package,
 installed next to `usersim`. It registers through entry points, so nothing in
-this repository changes. This guide builds one such package, `acme-recipes`: a
-probe that asks a cooking assistant questions from its own bank, and a scorer
-for the rows it produces.
+this repository changes. This guide builds one such package, `my-bookings`: a
+probe whose simulated users ask an assistant to book an appointment, writing the
+date the way people write dates in their own language, and a scorer that checks
+the booking the assistant makes.
 
 [`architecture/plugins.md`](../architecture/plugins.md) describes the entry
 point groups and how discovery behaves,
@@ -15,32 +16,35 @@ every name the package can import from `usersim`.
 ## Layout
 
 ```
-acme-recipes/
+my-bookings/
 ├── pyproject.toml
-├── src/acme_recipes/
+├── src/my_bookings/
 │   ├── __init__.py
 │   ├── probe.py
 │   ├── scorer.py
-│   └── assets/acme_recipes/en_US/recipes.yaml
+│   └── assets/my_bookings/
+│       ├── en_US/bookings.yaml
+│       └── pt_BR/bookings.yaml
 └── tests/
     ├── conftest.py
-    └── test_acme_recipes.py
+    └── test_my_bookings.py
 ```
 
-The banks sit under `assets/<label>/`, the layout the asset search path reads.
+The banks sit under `assets/<label>/<locale>/`, the layout the asset search path
+reads.
 
 ## Names
 
-Every name the package adds starts with its prefix, `acme_`, because unprefixed
+Every name the package adds starts with its prefix, `my_`, because unprefixed
 names belong to the built-ins:
 
-| Name | In `acme-recipes` |
+| Name | In `my-bookings` |
 |---|---|
-| Probe label and family | `acme_recipes` |
-| Scorer | `acme_recipes` |
-| Result column | `acme_recipe_id` |
-| Placeholder warning | `acme_used_placeholder_recipe` |
-| Metadata keys, if any | `acme_...` |
+| Probe label and family | `my_bookings` |
+| Scorer | `my_bookings` |
+| Result columns | `my_booking_id`, `my_booking` |
+| Placeholder warning | `my_used_placeholder_booking` |
+| Metadata key | `my_booking` |
 | Entry point names | the names they register |
 
 The conversation loop keeps its own record in `state.metadata` under the keys
@@ -55,19 +59,19 @@ requires = ["setuptools>=68"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "acme-recipes"
+name = "my-bookings"
 version = "0.1.0"
 requires-python = ">=3.10"
 dependencies = ["usersim>=X.Y,<X.Z", "pyyaml"]
 
 [project.entry-points."usersim.probes"]
-acme_recipes = "acme_recipes.probe:RecipeProbe"
+my_bookings = "my_bookings.probe:BookingProbe"
 
 [project.entry-points."usersim.scorers"]
-acme_recipes = "acme_recipes.scorer"
+my_bookings = "my_bookings.scorer"
 
 [tool.setuptools.package-data]
-acme_recipes = ["assets/**/*.yaml"]
+my_bookings = ["assets/**/*.yaml"]
 
 [dependency-groups]
 dev = ["pytest", "pytest-asyncio", "pytest-env"]
@@ -89,133 +93,262 @@ asyncio_mode = "auto"
 ## The probe
 
 ```python
+import json
 import random
 from dataclasses import dataclass
 from functools import cache
 from pathlib import Path
 
 import yaml
-
-from usersim.extension import BankBackedProbe, PreviewField, persona_uuid, register_probe
+from usersim.extension import (
+    BankBackedProbe,
+    BankVerbatimMixin,
+    PreviewField,
+    ToolExecutionMixin,
+    format_persona_for_prompt,
+    language_instruction,
+    persona_uuid,
+    register_probe,
+)
+
+#: The locales the package ships a bank for.
+LOCALES = ("en_US", "pt_BR")
+
+BOOK_APPOINTMENT = {
+    "type": "function",
+    "function": {
+        "name": "book_appointment",
+        "description": "Book an appointment.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "date": {"type": "string", "description": "YYYY-MM-DD"},
+                "time": {"type": "string", "description": "HH:MM, 24-hour"},
+                "people": {"type": "integer"},
+            },
+            "required": ["date", "time", "people"],
+        },
+    },
+}
 
 
 @dataclass(frozen=True)
-class Recipe:
+class Booking:
     id: str
-    question: str
+    opening: str
+    date: str
+    time: str
+    people: int
     placeholder: bool
 
 
 @dataclass(frozen=True)
-class RecipeBank:
+class BookingBank:
     bank_id: str
     bank_version: str
-    recipes: tuple[Recipe, ...]
+    bookings: tuple[Booking, ...]
 
 
-def read_recipe_bank(locale_dir: Path) -> RecipeBank:
-    doc = yaml.safe_load((locale_dir / "recipes.yaml").read_text(encoding="utf-8"))
-    return RecipeBank(doc["bank_id"], doc["bank_version"], tuple(Recipe(**r) for r in doc["recipes"]))
+def read_booking_bank(locale_dir: Path) -> BookingBank:
+    doc = yaml.safe_load((locale_dir / "bookings.yaml").read_text(encoding="utf-8"))
+    return BookingBank(doc["bank_id"], doc["bank_version"], tuple(Booking(**b) for b in doc["bookings"]))
 
 
 @cache
-def load_recipe_bank(locale: str) -> RecipeBank:
-    return read_recipe_bank(RecipeProbe.assets_dir() / locale)
-
-
-@register_probe(family="acme_recipes", prompt_version="v1.0", variants=("default",))
-class RecipeProbe(BankBackedProbe):
-    label = "acme_recipes"
-    description = "Tests cooking help: a home cook asks about a recipe from the bank."
-    preview_fields = (PreviewField("acme_recipe_id", "Recipe id"),)
-    default_scorers = ("acme_recipes",)
+def load_booking_bank(locale: str) -> BookingBank:
+    return read_booking_bank(BookingProbe.assets_dir() / locale)
+
+
+@register_probe(family="my_bookings", prompt_version="v1.0", variants=("default",))
+class BookingProbe(ToolExecutionMixin, BankVerbatimMixin, BankBackedProbe):
+    label = "my_bookings"
+    description = (
+        "Tests whether an assistant books what the user means: a date, a time and a number of people, "
+        "written the way people write them in the user's language."
+    )
+    preview_fields = (PreviewField("my_booking_id", "Booking id"),)
+    default_scorers = ("my_bookings",)
     assets_root = Path(__file__).parent / "assets"
-    bank_loader = staticmethod(load_recipe_bank)
-    locale_bank_check = staticmethod(read_recipe_bank)
-    placeholder_warning_kind = "acme_used_placeholder_recipe"
-    output_columns = ("acme_recipe_id",)
+    bank_loader = staticmethod(load_booking_bank)
+    locale_bank_check = staticmethod(read_booking_bank)
+    placeholder_warning_kind = "my_used_placeholder_booking"
+    verbatim_field = "opening"
+    output_columns = ("my_booking_id", "my_booking")
 
     @classmethod
     def asset_problems(cls) -> list[str]:
-        try:
-            read_recipe_bank(cls.assets_dir() / "en_US")
-        except Exception as error:
-            return [f"en_US: {type(error).__name__}: {error}"]
-        return []
+        problems = []
+        for locale in LOCALES:
+            try:
+                read_booking_bank(cls.assets_dir() / locale)
+            except Exception as error:
+                problems.append(f"{locale}: {type(error).__name__}: {error}")
+        return problems
 
     def derive_task(self, persona, bank, *, cfg):
         rng = random.Random(f"{persona_uuid(persona)}:{bank.bank_version}:{cfg.random_seed}")
-        return rng.choice(bank.recipes)
+        return rng.choice(bank.bookings)
 
     def get_user_system_prompt(self) -> str:
-        return f"You are a home cook. Ask a cooking assistant: {self._task.question}"
+        return "\n\n".join(
+            [
+                format_persona_for_prompt(self._persona),
+                "You asked an assistant to book an appointment, giving the date and time. "
+                f"If it asks how many people are coming, the answer is {self._task.people}; "
+                "say it the way you would. Once it confirms the booking, thank it.",
+                language_instruction(self._language, self._locale),
+            ]
+        )
 
     def get_assistant_system_prompt(self) -> str:
         return ""
 
+    def get_tools_for_assistant(self) -> list:
+        return [BOOK_APPOINTMENT]
+
+    def allow_early_stop_at_turn(self, turn_idx, state) -> bool:
+        return "my_booking" in state.metadata
+
+    async def execute_tool_call(self, name, args, tc, state, models, *, turn_idx, call_idx) -> str:
+        if "my_booking" not in state.metadata:
+            user_turns = sum(1 for message in state.messages if message.get("role") == "user")
+            state.metadata["my_booking"] = {"arguments": args, "after_user_turns": user_turns}
+        return json.dumps({"status": "booked"})
+
     def build_result_extras(self, state) -> dict:
-        return {**super().build_result_extras(state), "acme_recipe_id": self._task.id}
+        task = self._task
+        booking = {
+            "expected": {"date": task.date, "time": task.time, "people": task.people},
+            "made": state.metadata.get("my_booking"),
+        }
+        return {
+            **super().build_result_extras(state),
+            "my_booking_id": task.id,
+            "my_booking": json.dumps(booking, ensure_ascii=False),
+        }
 ```
 
+- **`ToolExecutionMixin`** runs the assistant's tool calls. The probe offers
+  `book_appointment` in `get_tools_for_assistant`, and `execute_tool_call`
+  answers each call. It records the first booking in `state.metadata`, with
+  how many user turns came before it.
+- **`BankVerbatimMixin`** sends the booking's `opening`, the field
+  `verbatim_field` names, as turn 1, so the date reaches the assistant written
+  exactly as the user wrote it.
+- **`get_user_system_prompt`** sets up the user model for the turns after the
+  first: the persona, the number of people it gives only when the assistant
+  asks, and `language_instruction`, which keeps it writing in the run's
+  language.
+- **`allow_early_stop_at_turn`** keeps the conversation going until the
+  assistant books, so a user who adds a thank-you to the number of people does
+  not end it before the booking is made.
+- **The assistant's system prompt is empty:** the tool's description is all the
+  assistant under test is told, so it answers as it would for any user.
 - **`description`** is what the per-row preview and the capability report
   show about the probe.
-- **`preview_fields`** puts the row's recipe id in the per-row preview's Probe
+- **`preview_fields`** puts the row's booking id in the per-row preview's Probe
   Inputs panel.
 - **`default_scorers`** names the package's scorer, so
-  `usersim eval --scorers auto` runs it whenever a run holds the probe's rows.
-- **`assets_root`** is the absolute directory holding `acme_recipes/`, so the
+  `usersim eval --scorers auto` runs it whenever a run holds the probe's rows,
+  beside `language_compliance`, which checks that the assistant replied in the
+  user's language.
+- **`assets_root`** is the absolute directory holding `my_bookings/`, so the
   probe finds its banks wherever the package is installed.
-- **`load_recipe_bank`** reads under `RecipeProbe.assets_dir()`, the probe's
+- **`load_booking_bank`** reads under `BookingProbe.assets_dir()`, the probe's
   folder on the asset search path, so a folder `--assets-dir` or
   `USERSIM_ASSET_PATH` names reaches it too.
 - **`locale_bank_check`** reads one locale's bank, so a run in a locale with no
   bank that loads stops before any model call rather than failing every row.
-- **`asset_problems`** reads the bank the package ships, so `usersim smoke` and
-  `assert_probe_conforms` report an install that is missing it.
-- **`placeholder_warning_kind`** is the warning a row records when its recipe
+- **`asset_problems`** reads each bank the package ships, so `usersim smoke` and
+  `assert_probe_conforms` report an install that is missing one.
+- **`placeholder_warning_kind`** is the warning a row records when its booking
   is marked `placeholder: true`. Its `<prefix>_used_placeholder_<thing>` form is
   what the capability report counts as placeholder content.
-- **`output_columns`** declares the column `build_result_extras` adds, so a run
-  keeps it.
-- **`derive_task`** picks the row's recipe with a generator seeded from the
+- **`output_columns`** declares the columns `build_result_extras` adds, so a run
+  keeps them. `my_booking` holds, as JSON, what the user meant and what the
+  assistant booked.
+- **`derive_task`** picks the row's booking with a generator seeded from the
   persona, the bank's version and the run's seed, never the global `random`, so
-  a row draws the same recipe every time it runs.
-- **The assistant's system prompt is empty:** the assistant under test answers
-  as it would for any user.
+  a row draws the same booking every time it runs.
+
+The banks, `assets/my_bookings/en_US/bookings.yaml`:
+
+```yaml
+bank_id: my_bookings_en_US
+bank_version: "1.0.0"
+bookings:
+  - id: museum_tour
+    opening: Hi! Can you book the guided museum tour for 04/03/2027 at 3:30 pm?
+    date: "2027-04-03"
+    time: "15:30"
+    people: 3
+    placeholder: true
+```
 
-The bank, `assets/acme_recipes/en_US/recipes.yaml`:
+and `assets/my_bookings/pt_BR/bookings.yaml`:
 
 ```yaml
-bank_id: acme_recipes
+bank_id: my_bookings_pt_BR
 bank_version: "1.0.0"
-recipes:
-  - id: pantry_curry
-    question: How do I make a chickpea curry with what is in my pantry?
+bookings:
+  - id: museum_tour
+    opening: Oi! Você pode reservar a visita guiada ao museu para 04/03/2027 às 15h30?
+    date: "2027-03-04"
+    time: "15:30"
+    people: 3
     placeholder: true
 ```
 
-`placeholder: true` marks content no subject-matter expert has reviewed; see
+Both open with the same date, `04/03/2027`. A user writing American English
+means 3 April and a user writing Brazilian Portuguese means 4 March. An
+assistant that reads every date month first books the Brazilian user's tour a
+month late, and one that reads every date day first books the American user's
+a month early. Neither opening says how many people are coming, so an assistant
+that books on its first reply has to guess.
+
+`placeholder: true` marks content no subject-matter expert has reviewed, which
+for these banks means a native speaker of the bank's language; see
 [`architecture/assets.md`](../architecture/assets.md).
 
 ## The scorer
 
 ```python
+import json
+
 from usersim.extension import register_scorer
 
 
 async def score(row, models):
-    """1.0 when the conversation ran against a recipe from the bank."""
-    return {"acme_recipe_grounded": 1.0 if row.get("acme_recipe_id") else 0.0}
-
-
-register_scorer("acme_recipes", score, deterministic=True)
+    """Whether the assistant booked what the user meant, after asking how many people."""
+    if not row.get("my_booking"):
+        return {}
+    booking = json.loads(row["my_booking"])
+    expected, made = booking["expected"], booking["made"]
+    if made is None:
+        return {"my_booked": 0.0}
+    arguments = made["arguments"]
+    return {
+        "my_booked": 1.0,
+        "my_date_correct": float(arguments.get("date") == expected["date"]),
+        "my_time_correct": float(arguments.get("time") == expected["time"]),
+        "my_people_correct": float(arguments.get("people") == expected["people"]),
+        "my_asked_before_booking": float(made["after_user_turns"] > 1),
+    }
+
+
+register_scorer("my_bookings", score, deterministic=True)
 ```
 
 The evaluator awaits a scorer once per trajectory, with the row and the
-models, and merges the mapping it returns into the row's evaluation. A field
-the row holds empty arrives as `None`. `deterministic=True` says the scorer
-never calls a model, so `usersim rescore` can recompute it on a stored
-evaluation.
+models, and merges the mapping it returns into the row's evaluation. It calls
+the scorer on every row it evaluates, so the scorer returns nothing for a row
+without a `my_booking` column, such as one from another probe. A field the
+row holds empty arrives as `None`. Each argument must match what the user meant
+exactly, in the form the tool's description asks for, and
+`my_asked_before_booking` is 1.0 when the booking came after the user's
+second turn, the first that can say how many people are coming.
+`deterministic=True` says the scorer never calls a model, so `usersim rescore`
+can recompute it on a stored evaluation.
 
 ## Tests
 
@@ -232,34 +365,53 @@ installed. With discovery off, the tests import the package to register it:
 ```python
 import json
 
-import acme_recipes.probe  # noqa: F401  registers the probe; discovery is off under test
-import acme_recipes.scorer  # noqa: F401  registers the scorer
 from usersim.extension import ConversationSimulatorConfig
 from usersim.testing import assert_hosted_parity, assert_probe_conforms, assert_scorer_conforms, run_episode
 
-PERSONA = {"first_name": "Sarah", "last_name": "Johnson", "age": 35, "city": "Seattle"}
-CONFIG = ConversationSimulatorConfig(name="conversation_messages", max_turns=2, random_seed=7)
+import my_bookings.probe  # noqa: F401  registers the probe; discovery is off under test
+from my_bookings.scorer import score
+
+PERSONA = {"first_name": "Ana", "last_name": "Souza", "age": 35, "city": "Recife"}
+CONFIG = ConversationSimulatorConfig(name="conversation_messages", max_turns=3, random_seed=7, locale="pt_BR")
 
 
 def test_the_probe_conforms():
-    assert_probe_conforms("acme_recipes")
+    assert_probe_conforms("my_bookings")
 
 
 async def test_the_scorer_conforms():
-    await assert_scorer_conforms("acme_recipes")
+    await assert_scorer_conforms("my_bookings")
 
 
-async def test_an_episode_runs_against_a_recipe():
-    run = await run_episode("acme_recipes", config=CONFIG, persona=PERSONA)
+async def test_an_episode_opens_in_portuguese_and_records_the_booking():
+    run = await run_episode("my_bookings", config=CONFIG, persona=PERSONA, language="Portuguese")
 
     assert run.row["conversation_status"] is True
-    assert run.row["acme_recipe_id"] == "pantry_curry"
+    opening = json.loads(run.row["conversation_messages"])[0]
+    assert opening["content"] == "Oi! Você pode reservar a visita guiada ao museu para 04/03/2027 às 15h30?"
+    booking = json.loads(run.row["my_booking"])
+    assert booking["expected"] == {"date": "2027-03-04", "time": "15:30", "people": 3}
+    assert booking["made"]["after_user_turns"] == 1  # the scripted assistant books on its first reply
     warnings = json.loads(run.row["simulation_outcome"])["warnings"]
-    assert [w["kind"] for w in warnings] == ["acme_used_placeholder_recipe"]
+    assert [w["kind"] for w in warnings] == ["my_used_placeholder_booking"]
+
+
+async def test_a_month_first_reading_of_a_day_first_date_is_wrong():
+    made = {"arguments": {"date": "2027-04-03", "time": "15:30", "people": 3}, "after_user_turns": 2}
+    expected = {"date": "2027-03-04", "time": "15:30", "people": 3}
+    result = await score({"my_booking": json.dumps({"expected": expected, "made": made})}, models={})
+
+    assert result == {
+        "my_booked": 1.0,
+        "my_date_correct": 0.0,
+        "my_time_correct": 1.0,
+        "my_people_correct": 1.0,
+        "my_asked_before_booking": 1.0,
+    }
 
 
 async def test_a_host_reproduces_a_standalone_run():
-    await assert_hosted_parity("acme_recipes", config=CONFIG, persona=PERSONA)
+    await assert_hosted_parity("my_bookings", config=CONFIG, persona=PERSONA, language="Portuguese")
 ```
 
 - **`assert_probe_conforms`** checks the probe's shape: its registration, its
@@ -267,7 +419,12 @@ async def test_a_host_reproduces_a_standalone_run():
 - **`assert_scorer_conforms`** checks that the scorer is async and tolerates
   empty fields.
 - **`run_episode`** runs a whole episode with scripted models, through the same
-  generator as `usersim simulate`, with nothing patched and no provider.
+  generator as `usersim simulate`, with nothing patched and no provider. Here it
+  runs in `pt_BR`: the opening reaches the assistant as written, and the
+  scripted assistant, which calls a tool as soon as one is offered, books on its
+  first reply.
+- **The scorer** is tested on a row built by hand, in which the assistant read
+  the Brazilian date month first.
 - **`assert_hosted_parity`** checks that a host driving the probe through
   `ProbeEpisodeRuntime` reproduces a standalone run.
 
@@ -283,7 +440,7 @@ Outside the tests, discovery is on, so `usersim smoke` lists each installed
 extension package with its entry points, in a line like this one:
 
 ```
-[OK]   extensions            acme-recipes 0.1.0: 2 entry point(s)
+[OK]   extensions            my-bookings 0.1.0: 2 entry point(s)
 ```
 
 It fails when an entry point contributes nothing: it failed to load, another
@@ -295,11 +452,13 @@ Once installed, the probe joins a run by its label. This makes live model calls
 and is billed by the provider:
 
 ```bash
-usersim simulate --locale en_US --num-rows 5 \
-  --probe-mix "acme_recipes=1.0" --out output/trajectories
+usersim simulate --locale pt_BR --num-rows 5 \
+  --probe-mix "my_bookings=1.0" --out output/trajectories
 ```
 
-The package has a bank for `en_US` only, so a run with `--locale fr_FR` stops
-before any model call and says the probe has no bank for it. `--assets-dir`
-pins every probe in a run to the folder it names, this one included, so a run
-that passes it needs `acme_recipes/` there too.
+The package has banks for `en_US` and `pt_BR` only, so a run with
+`--locale fr_FR` stops before any model call and says the probe has no bank for
+it. A new locale is a folder holding a `bookings.yaml` written in its language,
+plus an entry in `LOCALES`. `--assets-dir` pins every probe in a run to the
+folder it names, this one included, so a run that passes it needs
+`my_bookings/` there too.
diff --git a/tests/test_extension_api.py b/tests/test_extension_api.py
index 64e35a5..e965bb9 100644
--- a/tests/test_extension_api.py
+++ b/tests/test_extension_api.py
@@ -53,11 +53,11 @@
 #: Imports from other ``usersim`` modules that the examples keep, as (module, name).
 _KEPT_INTERNAL_IMPORTS = {
     # A new in-tree probe's own modules, under the names the examples give them.
-    ("usersim.engine.core.cooking_bank", "CookingRecipe"),
-    ("usersim.engine.core.cooking_bank", "load_cooking_bank"),
     ("usersim.engine.probes.YOUR_PROBE", "generator"),
-    ("usersim.engine.probes.cooking_advisor.prompts", "COOKING_GATE_PROMPT"),
-    ("usersim.engine.probes.cooking_advisor.prompts", "COOKING_USER_PROMPTS"),
+    ("usersim.engine.probes.bookings.bank", "LOCALES"),
+    ("usersim.engine.probes.bookings.bank", "Booking"),
+    ("usersim.engine.probes.bookings.bank", "read_booking_bank"),
+    ("usersim.engine.probes.bookings.prompts", "BOOKINGS_USER_PROMPTS"),
     ("usersim.engine.probes.my_probe.bank", "load_my_bank_default"),
     ("usersim.engine.templates.probe.prompts", "DEMO_GATE_PROMPT_TEMPLATE"),
     ("usersim.engine.templates.probe.prompts", "DEMO_USER_AGENT_PROMPTS"),

From 40bcc1679ba5ba602e69eddb04ca12b444c071cf Mon Sep 17 00:00:00 2001
From: Yev Meyer <ymeyer@nvidia.com>
Date: Sun, 1 Nov 2026 15:00:03 -0500
Subject: [PATCH 6/6] refactor(engine): use my_ as the example extension prefix

Every example of a name an extension adds starts with my_, the prefix of the package docs/extensions.md builds: the warning kind in architecture/plugins.md and the authoring guide, the example in the error for a malformed warning kind, and the warning kinds, metadata keys and scorer names in the tests that stand in for an extension.

Signed-off-by: Yev Meyer <ymeyer@nvidia.com>
---
 architecture/plugins.md                    |  2 +-
 docs/engine/AUTHORING_A_PROBE.md           |  2 +-
 src/usersim/engine/core/outcomes.py        |  6 +++---
 tests/engine/core/test_outcomes.py         | 16 ++++++++--------
 tests/engine/core/test_preview.py          |  2 +-
 tests/engine/core/test_probes_substrate.py |  4 ++--
 tests/engine/test_conformance_suite.py     |  8 ++++----
 tests/test_capability_report.py            |  2 +-
 tests/test_rescore.py                      |  8 ++++----
 9 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/architecture/plugins.md b/architecture/plugins.md
index 705fb2f..c0b2e01 100644
--- a/architecture/plugins.md
+++ b/architecture/plugins.md
@@ -55,7 +55,7 @@ per-row data goes in its metadata (`state.metadata`), stored with the row as
 An extension's warnings carry its prefix too: a bank-backed probe names its
 placeholder warning `<prefix>_used_placeholder_<thing>`, which the capability
 report counts as placeholder content, and any other warning it records is a
-prefixed name such as `acme_slow_tool`.
+prefixed name such as `my_slow_tool`.
 
 ## How discovery behaves
 
diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md
index 08b3c39..6d1b81b 100644
--- a/docs/engine/AUTHORING_A_PROBE.md
+++ b/docs/engine/AUTHORING_A_PROBE.md
@@ -202,7 +202,7 @@ form `<prefix>_used_placeholder_<thing>`, such as
 `"my_used_placeholder_booking"`; the report counts it like the built-in kinds.
 `@register_probe` refuses a kind that does not mark placeholder content. Other
 warnings a probe adds with `add_warning` take the same prefixed form, such as
-`"acme_slow_tool"`.
+`"my_slow_tool"`.
 
 For `BaseProbe`-only subclasses, just `label` is required.
 
diff --git a/src/usersim/engine/core/outcomes.py b/src/usersim/engine/core/outcomes.py
index 682791d..bddafae 100644
--- a/src/usersim/engine/core/outcomes.py
+++ b/src/usersim/engine/core/outcomes.py
@@ -174,7 +174,7 @@ class WarningKind(str, Enum):
 )
 
 #: A warning kind a probe outside this package defines: lowercase words joined
-#: by underscores, led by the package's own prefix, such as ``acme_slow_tool``.
+#: by underscores, led by the package's own prefix, such as ``my_slow_tool``.
 _EXTENSION_WARNING_KIND = re.compile(r"[a-z][a-z0-9]*(?:_[a-z0-9]+)+")
 
 
@@ -182,13 +182,13 @@ def warning_kind_problem(kind: object) -> str | None:
     """Why ``kind`` cannot be a warning kind, or ``None`` when it can.
 
     A ``WarningKind`` member, or a string: either a built-in kind's value or a
-    prefixed lowercase name such as ``acme_slow_tool``.
+    prefixed lowercase name such as ``my_slow_tool``.
     """
     if isinstance(kind, WarningKind):
         return None
     if isinstance(kind, str) and _EXTENSION_WARNING_KIND.fullmatch(kind):
         return None
-    return f"must be a WarningKind or a prefixed lowercase name such as 'acme_slow_tool', got {kind!r}"
+    return f"must be a WarningKind or a prefixed lowercase name such as 'my_slow_tool', got {kind!r}"
 
 
 def is_placeholder_warning_kind(kind: object) -> bool:
diff --git a/tests/engine/core/test_outcomes.py b/tests/engine/core/test_outcomes.py
index 9450b1c..555c886 100644
--- a/tests/engine/core/test_outcomes.py
+++ b/tests/engine/core/test_outcomes.py
@@ -156,13 +156,13 @@ def test_a_built_in_value_becomes_its_member(self) -> None:
 
     def test_a_prefixed_string_kind_is_stored_as_given(self) -> None:
         builder = OutcomeBuilder()
-        builder.add_warning(kind="acme_slow_tool", turn_idx=2, detail="tool took 30 s")
+        builder.add_warning(kind="my_slow_tool", turn_idx=2, detail="tool took 30 s")
 
         (warning,) = builder.finalize(OutcomeStatus.OK).to_dict()["warnings"]
 
-        assert warning == {"kind": "acme_slow_tool", "turn_idx": 2, "detail": "tool took 30 s"}
+        assert warning == {"kind": "my_slow_tool", "turn_idx": 2, "detail": "tool took 30 s"}
 
-    @pytest.mark.parametrize("kind", ["slowtool", "Acme_slow_tool", "acme slow tool", "acme_", 5])
+    @pytest.mark.parametrize("kind", ["slowtool", "My_slow_tool", "my slow tool", "my_", 5])
     def test_an_unusable_kind_is_refused(self, kind) -> None:
         with pytest.raises(ValueError, match="prefixed lowercase name"):
             SimulationWarning(kind=kind)
@@ -171,8 +171,8 @@ def test_an_unusable_kind_is_refused(self, kind) -> None:
         ("kind", "placeholder"),
         [
             *[(member, member.name.startswith("USED_PLACEHOLDER_")) for member in WarningKind],
-            ("acme_used_placeholder_recipe", True),
-            ("acme_slow_tool", False),
+            ("my_used_placeholder_booking", True),
+            ("my_slow_tool", False),
             ("used_placeholder_recipe", False),
             (None, False),
         ],
@@ -186,10 +186,10 @@ def test_placeholder_kinds_are_recognised(self, kind, placeholder: bool) -> None
 
     def test_a_placeholder_warning_kind_must_mark_placeholder_content(self) -> None:
         assert placeholder_warning_kind_problem(WarningKind.USED_PLACEHOLDER_FACT) is None
-        assert placeholder_warning_kind_problem("acme_used_placeholder_recipe") is None
+        assert placeholder_warning_kind_problem("my_used_placeholder_booking") is None
         assert "must mark placeholder content" in placeholder_warning_kind_problem(WarningKind.ID_FABRICATION)
-        assert "must mark placeholder content" in placeholder_warning_kind_problem("acme_slow_tool")
-        assert "prefixed lowercase name" in placeholder_warning_kind_problem("Acme")
+        assert "must mark placeholder content" in placeholder_warning_kind_problem("my_slow_tool")
+        assert "prefixed lowercase name" in placeholder_warning_kind_problem("My")
 
 
 class TestBuilderCounters:
diff --git a/tests/engine/core/test_preview.py b/tests/engine/core/test_preview.py
index 5456b41..e5a8206 100644
--- a/tests/engine/core/test_preview.py
+++ b/tests/engine/core/test_preview.py
@@ -546,7 +546,7 @@ def test_every_placeholder_kind_reads_as_ok(self, kind: str) -> None:
         assert kind not in out
 
     @pytest.mark.parametrize(
-        ("kind", "status"), [("acme_used_placeholder_recipe", "SIM OK"), ("acme_slow_tool", "SIM WARN")]
+        ("kind", "status"), [("my_used_placeholder_booking", "SIM OK"), ("my_slow_tool", "SIM WARN")]
     )
     def test_a_prefixed_kind_reads_by_whether_it_marks_placeholder_content(self, kind: str, status: str) -> None:
         row = _general_row()
diff --git a/tests/engine/core/test_probes_substrate.py b/tests/engine/core/test_probes_substrate.py
index ac50c51..e331b33 100644
--- a/tests/engine/core/test_probes_substrate.py
+++ b/tests/engine/core/test_probes_substrate.py
@@ -890,7 +890,7 @@ def test_rejects_an_output_column_the_engine_writes(self) -> None:
     def test_accepts_a_prefixed_placeholder_warning_kind(self) -> None:
         mod = self._make_module()
         bank_probe = self._probe_class("_Recipes", "_recipe_bank_probe", mod.__name__)
-        bank_probe.placeholder_warning_kind = "acme_used_placeholder_recipe"
+        bank_probe.placeholder_warning_kind = "my_used_placeholder_booking"
 
         registered = register_probe(family="general", prompt_version="v1.0", variants=("default",))(bank_probe)
 
@@ -900,7 +900,7 @@ def test_rejects_a_placeholder_warning_kind_that_marks_no_placeholder(self) -> N
         """The report counts placeholder rows by kind, so any other kind would hide them."""
         mod = self._make_module()
         bank_probe = self._probe_class("_Mislabelled", "_mislabelled_bank_probe", mod.__name__)
-        bank_probe.placeholder_warning_kind = "acme_slow_tool"
+        bank_probe.placeholder_warning_kind = "my_slow_tool"
 
         with pytest.raises(ProbeRegistrationError, match="placeholder_warning_kind must mark placeholder content"):
             register_probe(family="general", prompt_version="v1.0", variants=("default",))(bank_probe)
diff --git a/tests/engine/test_conformance_suite.py b/tests/engine/test_conformance_suite.py
index 56c6135..9c2d1f6 100644
--- a/tests/engine/test_conformance_suite.py
+++ b/tests/engine/test_conformance_suite.py
@@ -62,9 +62,9 @@ class _WritesItsOwnKeys(BaseProbe):
     label = "writes_its_own_keys"
 
     def seed_state_metadata(self, state):
-        state.metadata["acme_checks"] = []
-        state.metadata.update({"acme_seen": True})
-        state.metadata.setdefault("acme_turns", 0)
+        state.metadata["my_checks"] = []
+        state.metadata.update({"my_seen": True})
+        state.metadata.setdefault("my_turns", 0)
 
 
 #: A row shaped like one the evaluator hands a scorer. Passing it makes the
@@ -172,7 +172,7 @@ async def test_a_placeholder_warning_kind_that_marks_no_placeholder_is_reported(
 
         class Mislabelled(BaseProbe):
             label = "mislabelled_probe"
-            placeholder_warning_kind = "acme_slow_tool"
+            placeholder_warning_kind = "my_slow_tool"
 
         problems = probe_conformance_problems(Mislabelled)
         assert any("Mislabelled.placeholder_warning_kind must mark placeholder content" in p for p in problems), (
diff --git a/tests/test_capability_report.py b/tests/test_capability_report.py
index d79ae83..61fabb8 100644
--- a/tests/test_capability_report.py
+++ b/tests/test_capability_report.py
@@ -358,7 +358,7 @@ def test_every_placeholder_warning_counts_as_placeholder_content(kind):
     assert (cell.n_placeholder, cell.n_machine_translated) == (1, 0)
 
 
-@pytest.mark.parametrize(("kind", "placeholder"), [("acme_used_placeholder_recipe", 1), ("acme_slow_tool", 0)])
+@pytest.mark.parametrize(("kind", "placeholder"), [("my_used_placeholder_booking", 1), ("my_slow_tool", 0)])
 def test_a_prefixed_kind_counts_as_placeholder_content_only_when_it_says_so(kind, placeholder):
     """A probe outside this package names its placeholder kind ``<prefix>_used_placeholder_<thing>``."""
     cell = _quality_cell(_flagged_row(0, kind))
diff --git a/tests/test_rescore.py b/tests/test_rescore.py
index 9e715dc..6b2ca98 100644
--- a/tests/test_rescore.py
+++ b/tests/test_rescore.py
@@ -223,14 +223,14 @@ def test_hybrid_scorers_are_refused(self, run_on_disk, scorer):
     def test_a_scorer_registered_without_the_flag_is_refused(self, run_on_disk, monkeypatch):
         """Rescore reruns only what a scorer's registration promises calls no model."""
 
-        async def acme_judged(traj, models):
+        async def my_judged(traj, models):
             return {"scores": {}}
 
-        _register_for_this_test(monkeypatch, "acme_judged", acme_judged)
-        run_on_disk.scorers = ["acme_judged"]
+        _register_for_this_test(monkeypatch, "my_judged", my_judged)
+        run_on_disk.scorers = ["my_judged"]
         with pytest.raises(SystemExit) as excinfo:
             run(run_on_disk)
-        assert "'acme_judged' is not registered as deterministic" in str(excinfo.value)
+        assert "'my_judged' is not registered as deterministic" in str(excinfo.value)
 
     def test_unknown_scorer_is_refused(self, run_on_disk):
         run_on_disk.scorers = ["not_a_scorer"]