From a25754c653ed1a46723554c21ea45f6a382684ca Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Mon, 2 Nov 2026 16:46:45 -0500 Subject: [PATCH 1/7] fix(engine): read prompts.yaml from the folder a probe's banks resolve to load_prompt_file reads a probe's prompts.yaml from the folder probe_assets_dir resolves for it, as the probe's bank loaders do: the first root on the asset search path that holds the probe, or the assets_root its class declares. A root on USERSIM_ASSET_PATH that holds other probes' banks does not hide a probe's prompts, and a probe that declares assets_root has its prompts.yaml read from there. A test covers each case. Signed-off-by: Yev Meyer --- architecture/assets.md | 3 ++- docs/engine/README.md | 2 +- src/usersim/engine/core/prompt_loader.py | 24 +++++++++++++-------- tests/engine/core/test_prompt_resolution.py | 22 +++++++++++++++++-- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/architecture/assets.md b/architecture/assets.md index 8986f98c..852969c4 100644 --- a/architecture/assets.md +++ b/architecture/assets.md @@ -31,7 +31,8 @@ set. Registration refuses a relative root. That root is searched for that probe only, after the first three entries and before the packaged directory, so a deliberate override wins over it. The built-in probes declare none. A probe's bank loader reads under its class's `assets_dir()`, the folder this -search finds for the probe. +search finds for the probe, and its `prompts.yaml` is read from that folder +too. An explicit `--assets-dir` is **not** part of the search. It pins resolution to exactly that directory, for every probe in the run, including one that diff --git a/docs/engine/README.md b/docs/engine/README.md index aae70237..91046df8 100644 --- a/docs/engine/README.md +++ b/docs/engine/README.md @@ -528,7 +528,7 @@ what each one carries: | `core/persona.py` | Persona formatting for prompts (deterministic shuffle) | | `core/locale.py` | Locale → language + expected-script-ranges mapping (single source of truth for shipped locales) | | `core/localized.py` | `LocalizedText` wrapper for asset-bank fields with per-locale renderings (`for_locale(locale)` lookup with `en_US` fallback) | -| `core/prompt_loader.py` | Reads per-probe prompt packs from `assets//prompts.yaml` | +| `core/prompt_loader.py` | Reads a probe's `prompts.yaml` from the folder its banks resolve to | | `core/messages.py` | Message, tool, theme formatting | | `core/preview.py` | Rich-based preview utilities for inspecting generated conversations | diff --git a/src/usersim/engine/core/prompt_loader.py b/src/usersim/engine/core/prompt_loader.py index 11de289b..ca1e3e35 100644 --- a/src/usersim/engine/core/prompt_loader.py +++ b/src/usersim/engine/core/prompt_loader.py @@ -4,15 +4,17 @@ """YAML-based prompt template loader. Loads prompt templates from per-probe YAML files at -``assets//prompts.yaml``, falling back to the compiled-in +``//prompts.yaml``, falling back to the compiled-in Python defaults if YAML files are not found. This allows prompt iteration and A/B testing without code changes. -Per-probe asset layout: each probe owns its own folder under -``assets/`` and stores its prompt templates as ``prompts.yaml`` -inside that folder, alongside its other assets (e.g. -``assets/tool_calling/prompts.yaml`` lives next to -``assets/tool_calling/toolsets_seed.parquet``). +Per-probe asset layout: each probe owns a ``/`` folder under an +asset root and stores its prompt templates as ``prompts.yaml`` inside +that folder, alongside its other assets (e.g. +``tool_calling/prompts.yaml`` lives next to +``tool_calling/toolsets_seed.parquet``). The file is read from the +folder :func:`~usersim.engine.core._assets.probe_assets_dir` resolves, +the one the probe's banks are read from. """ from __future__ import annotations @@ -46,15 +48,19 @@ def _load_prompt_file_at(probe: str, assets_root: str) -> dict[str, Any]: def load_prompt_file(probe: str) -> dict[str, Any]: - """Load a probe's ``prompts.yaml`` from the active asset root. + """Load a probe's ``prompts.yaml`` from the folder its banks resolve to. + + That is the first root on the asset search path holding ``probe``, or + the ``assets_root`` its class declares, so a root that holds other + probes' assets does not hide this probe's prompts. Returns an empty dict if the file doesn't exist or PyYAML is not available — callers should always pass a sensible compiled-in default to :func:`get_prompt` to cover that path. """ - from usersim.engine.core._assets import default_assets_dir + from usersim.engine.core._assets import probe_assets_dir - return _load_prompt_file_at(probe, str(default_assets_dir())) + return _load_prompt_file_at(probe, str(probe_assets_dir(probe).parent)) def get_prompt(probe: str, key: str, default: str = "") -> str: diff --git a/tests/engine/core/test_prompt_resolution.py b/tests/engine/core/test_prompt_resolution.py index 68c52ad8..317d0025 100644 --- a/tests/engine/core/test_prompt_resolution.py +++ b/tests/engine/core/test_prompt_resolution.py @@ -25,8 +25,8 @@ import pytest import yaml -from usersim.engine.core._assets import packaged_assets_dir, set_runtime_assets_dir -from usersim.engine.core.prompt_loader import render_prompt +from usersim.engine.core._assets import packaged_assets_dir, probe_assets_dir, set_runtime_assets_dir +from usersim.engine.core.prompt_loader import get_prompt, load_prompt_file, render_prompt #: The theme-driven probes, mapped to the name each one gives the #: import-time constant behind ``user_agent_system_prompt()``. Spelled out @@ -155,6 +155,24 @@ def test_missing_key_falls_back_to_the_compiled_default(self, tmp_path): assert oe.trajectory_rubric() == "R" +class TestPromptFileIsReadWhereTheBanksAre: + """A probe's ``prompts.yaml`` comes from the folder its banks resolve to.""" + + def test_a_root_holding_other_probes_does_not_hide_the_shipped_prompts(self, tmp_path): + set_runtime_assets_dir(_assets_with_prompt(tmp_path, "tool_calling", user_agent_system_prompt="OTHER")) + + shipped = yaml.safe_load((probe_assets_dir("general_open_ended") / "prompts.yaml").read_text()) + assert shipped + assert load_prompt_file("general_open_ended") == shipped + + def test_a_declared_root_supplies_its_probes_prompts(self, register_test_probe, tmp_path): + register_test_probe( + "ext_probe", assets_root=_assets_with_prompt(tmp_path, "ext_probe", user_agent_system_prompt="EXT") + ) + + assert get_prompt("ext_probe", "user_agent_system_prompt") == "EXT" + + # --------------------------------------------------------------------------- # Per-row wiring # --------------------------------------------------------------------------- From 711019c282c85718d31a3c7b35eaa67ece935a23 Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Mon, 2 Nov 2026 16:47:02 -0500 Subject: [PATCH 2/7] build: ship the banks a probe keeps in its own folder Package data includes the files under probes/