From 11da5ceea60f206c8cc61ad8759df078dba9bf18 Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 3 Nov 2026 09:46:34 -0500 Subject: [PATCH 1/5] test: check the probe template's worked examples by their paths The probe test template cites each worked example by its path from the repo root, and the test of those examples checks that every cited path is a file, so an example kept in tests/probes/ is checked as one in tests/engine/probes/ is. Signed-off-by: Yev Meyer --- templates/probe/test_probe.py.template | 4 ++-- tests/test_packaging_contract.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/probe/test_probe.py.template b/templates/probe/test_probe.py.template index 06c78c4..802b3d7 100644 --- a/templates/probe/test_probe.py.template +++ b/templates/probe/test_probe.py.template @@ -12,9 +12,9 @@ after it. For real working examples, see: - ``tests/probes/sov_ai_facts/test_probe.py`` — banked-verbatim probe (the canonical reference for the alias-dispatching mock + dual-module patching pattern). -- ``test_safety_chat_pressure.py`` — bank-reframing probe +- ``tests/engine/probes/test_safety_chat_pressure.py`` — bank-reframing probe (``on_followup_failure="abort"`` test case). -- ``test_safety_agentic.py`` — agentic probe (custom ``run_dispatch``, +- ``tests/engine/probes/test_safety_agentic.py`` — agentic probe (custom ``run_dispatch``, no user-LLM, simpler mock pattern — FIFO-only since there's only one acall_llm caller). diff --git a/tests/test_packaging_contract.py b/tests/test_packaging_contract.py index f0b9959..d557de3 100644 --- a/tests/test_packaging_contract.py +++ b/tests/test_packaging_contract.py @@ -251,9 +251,9 @@ def test_referenced_examples_exist(self) -> None: import re src = self.TEMPLATE.read_text() - cited = set(re.findall(r"``(test_\w+)\.py``", src)) + cited = set(re.findall(r"``(tests/[\w/]+\.py)``", src)) assert cited, "template should cite worked examples" - missing = [name for name in cited if not (_REPO_ROOT / "tests" / "engine" / "probes" / f"{name}.py").is_file()] + missing = [path for path in cited if not (_REPO_ROOT / path).is_file()] assert not missing, f"template cites tests that do not exist: {sorted(missing)}" From efefe61efd24a69236ce86e2b79090611db9ddf4 Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 3 Nov 2026 09:46:39 -0500 Subject: [PATCH 2/5] refactor(engine): keep safety_chat_pressure in one folder The probe module is probes/safety_chat_pressure/probe.py, beside its classifier, its bank loader bank.py, its scorer scorer.py and its bank in assets/safety_chat_pressure/, the folder the class names as its assets_root. Its tests are in tests/probes/safety_chat_pressure/. The label, the scorer name, the bank, the stored columns and the cache the probe and its scorer share are unchanged. DEFAULT_SCORER_MODULES, the capability source references and the docs name the new modules, the authoring guide lists the probe among those kept in one folder, and the shared-code label allowances list neither the bank loader nor the scorer. Signed-off-by: Yev Meyer --- docs/engine/AUTHORING_A_PROBE.md | 10 +-- docs/engine/README.md | 10 +-- src/usersim/engine/core/agentic_bank.py | 2 +- src/usersim/engine/core/probes.py | 2 +- .../engine/evaluator/scorers/__init__.py | 2 +- .../probes/safety_chat_pressure/__init__.py | 2 +- .../assets/safety_chat_pressure/SCHEMA.md | 4 +- .../assets/safety_chat_pressure/sample.yaml | 0 .../safety_chat_pressure/bank.py} | 8 +-- .../{generator.py => probe.py} | 16 +++-- .../probes/safety_chat_pressure/prompts.py | 2 +- .../safety_chat_pressure/scorer.py} | 22 +++---- .../safety_chat_pressure/task_derivation.py | 2 +- src/usersim/taxonomy/capabilities.py | 4 +- templates/probe/test_probe.py.template | 2 +- tests/engine/core/test_agentic_bank.py | 2 +- tests/engine/core/test_assets_discovery.py | 4 +- tests/engine/core/test_default_asset_paths.py | 8 +-- ...luator_scorer_structured_output_schemas.py | 4 +- .../probes/test_india_variant_probes.py | 2 +- tests/engine/probes/test_safety_agentic.py | 2 +- tests/probes/safety_chat_pressure/__init__.py | 2 + .../safety_chat_pressure/test_bank.py} | 8 +-- .../safety_chat_pressure/test_classifier.py} | 0 .../safety_chat_pressure/test_probe.py} | 24 +++---- .../safety_chat_pressure/test_scorer.py} | 66 +++++++++---------- tests/test_cli_simulate_inputs.py | 4 +- tests/test_probe_names_in_shared_code.py | 2 - 28 files changed, 109 insertions(+), 107 deletions(-) rename src/usersim/engine/{ => probes/safety_chat_pressure}/assets/safety_chat_pressure/SCHEMA.md (98%) rename src/usersim/engine/{ => probes/safety_chat_pressure}/assets/safety_chat_pressure/sample.yaml (100%) rename src/usersim/engine/{core/pressure_bank.py => probes/safety_chat_pressure/bank.py} (98%) rename src/usersim/engine/probes/safety_chat_pressure/{generator.py => probe.py} (99%) rename src/usersim/engine/{evaluator/scorers/safety_chat_pressure.py => probes/safety_chat_pressure/scorer.py} (97%) create mode 100644 tests/probes/safety_chat_pressure/__init__.py rename tests/{engine/core/test_pressure_bank.py => probes/safety_chat_pressure/test_bank.py} (98%) rename tests/{engine/probes/test_safety_chat_pressure_classifier.py => probes/safety_chat_pressure/test_classifier.py} (100%) rename tests/{engine/probes/test_safety_chat_pressure.py => probes/safety_chat_pressure/test_probe.py} (99%) rename tests/{engine/evaluator/test_evaluator_scorer_safety_chat_pressure.py => probes/safety_chat_pressure/test_scorer.py} (92%) diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md index 2d945db..b0966e7 100644 --- a/docs/engine/AUTHORING_A_PROBE.md +++ b/docs/engine/AUTHORING_A_PROBE.md @@ -257,10 +257,10 @@ class MyProbe(BaseProbe): It must be absolute: `@register_probe` refuses a relative path, which would resolve against the working directory. In this repository the banks are then at `src/usersim/engine/probes/