From 927815d9d6cc600c93335dbda4de05fd102f79be Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Wed, 4 Nov 2026 11:50:42 -0500 Subject: [PATCH 1/9] refactor(engine): keep tool_calling's themes in its assets tool_calling's themes live in assets/tool_calling/en_US/themes.json, which every locale reads unless it has a themes.json of its own, and the probe's themes() reads them with load_seeds, as the general probes read their topics. The run setup draws them as it draws every other probe's, so build_simulator_config_builder and materialize_episode_inputs take no tool_calling_themes, and the run manifest's scope records none; a manifest that holds the key still loads. The README says the themes reach a row only when its toolset has no category, since the probe takes a row's theme from its toolset's category and name. asset_problems checks the themes file, and notebook 01 follows. Signed-off-by: Yev Meyer --- notebooks/01_simulate.ipynb | 3 +- src/usersim/cli/_pipeline.py | 21 +--------- src/usersim/cli/simulate.py | 6 +-- src/usersim/engine/core/manifest.py | 6 --- .../assets/tool_calling/README.md | 7 ++-- .../assets/tool_calling/en_US/themes.json | 42 +++++++++++++++++++ .../engine/probes/tool_calling/probe.py | 12 +++++- tests/engine/core/test_manifest.py | 12 +++++- 8 files changed, 70 insertions(+), 39 deletions(-) create mode 100644 src/usersim/engine/probes/tool_calling/assets/tool_calling/en_US/themes.json diff --git a/notebooks/01_simulate.ipynb b/notebooks/01_simulate.ipynb index 911db652..90c02b0b 100644 --- a/notebooks/01_simulate.ipynb +++ b/notebooks/01_simulate.ipynb @@ -669,7 +669,7 @@ "source": [ "from usersim.engine.config import ConversationSimulatorConfig\n", "from usersim.engine.core.manifest import write_simulator_manifest\n", - "from usersim.cli._pipeline import DEFAULT_TOOL_CALLING_THEMES, verify_probe_assets\n", + "from usersim.cli._pipeline import verify_probe_assets\n", "\n", "total_written = 0\n", "STARTED_AT = time.time()\n", @@ -755,7 +755,6 @@ " sim_config=sim_cfg_for_manifest,\n", " locales_requested=_run_locales,\n", " probe_mix=PROBE_MIX,\n", - " tool_calling_themes=DEFAULT_TOOL_CALLING_THEMES,\n", " toolset_seed_path=TOOLSET_SEED_PATH if TOOLSET_SEED_PATH.exists() else None,\n", " assets_dir=ASSETS_DIR,\n", " trajectories_requested={l: NUM_ROWS_PER_LOCALE for l in _run_locales},\n", diff --git a/src/usersim/cli/_pipeline.py b/src/usersim/cli/_pipeline.py index faa8a68f..2798f687 100644 --- a/src/usersim/cli/_pipeline.py +++ b/src/usersim/cli/_pipeline.py @@ -54,17 +54,6 @@ logger = logging.getLogger("usersim.cli.pipeline") -DEFAULT_TOOL_CALLING_THEMES: tuple[str, ...] = ( - '{"type": "Weather & Location Lookup", "description": "Look up current weather, forecasts, or location-based info.", "tool_expected": true}', - '{"type": "Calendar & Scheduling", "description": "Create, modify, or check calendar events and availability.", "tool_expected": true}', - '{"type": "Data Retrieval & Search", "description": "Search databases, look up records, or retrieve structured data.", "tool_expected": true}', - '{"type": "Financial Operations", "description": "Check balances, make transactions, or calculate financial metrics.", "tool_expected": true}', - '{"type": "Communication & Messaging", "description": "Send emails, messages, or notifications on behalf of the user.", "tool_expected": true}', - '{"type": "Multi-Step Workflow", "description": "Complete a task requiring multiple sequential tool calls.", "tool_expected": true}', - '{"type": "General Knowledge Question", "description": "Answer a factual question that requires no tools.", "tool_expected": false}', - '{"type": "Opinion or Advice Request", "description": "Ask for subjective advice or opinions where tools are unnecessary.", "tool_expected": false}', -) - DEFAULT_PROBE_MIX: dict[str, float] = { "tool_calling": 1 / 3, "general_open_ended": 1 / 3, @@ -441,7 +430,6 @@ def build_simulator_config_builder( max_assistant_attempts: int = 1, store_reasoning: bool = True, probe_mix: dict[str, float] | None = None, - tool_calling_themes: tuple[str, ...] | None = None, toolset_seed_path: Path | None = None, match_persona_language: bool = False, verbosity: int = 1, @@ -475,7 +463,6 @@ def build_simulator_config_builder( locale=locale, assets_dir=assets_dir, probe_mix=probe_mix, - tool_calling_themes=tool_calling_themes, toolset_seed_path=toolset_seed_path, match_persona_language=match_persona_language, drop_persona=True, @@ -514,7 +501,6 @@ def _add_episode_input_columns( locale: str, assets_dir: Path | None, probe_mix: dict[str, float], - tool_calling_themes: tuple[str, ...] | None, toolset_seed_path: Path | None, match_persona_language: bool, drop_persona: bool, @@ -551,12 +537,9 @@ def _add_episode_input_columns( themes: dict[str, list[str]] = {} toolset_kwargs: dict[str, Any] = {} for probe in probe_mix: + themes[probe] = _probe_themes(probe, locale, assets_dir) if probe == "tool_calling": - # Its themes and toolsets are settings of the run. - themes[probe] = list(tool_calling_themes or DEFAULT_TOOL_CALLING_THEMES) toolset_kwargs = _wire_toolsets(config_builder, dd, resolved_toolset_seed) - else: - themes[probe] = _probe_themes(probe, locale, assets_dir) config_builder.add_column( dd.SamplerColumnConfig( name="theme", @@ -575,7 +558,6 @@ def materialize_episode_inputs( probe_mix: dict[str, float], assets_dir: Path | None = None, random_seed: int | None = None, - tool_calling_themes: tuple[str, ...] | None = None, toolset_seed_path: Path | None = None, match_persona_language: bool = False, max_turns: int = 5, @@ -600,7 +582,6 @@ def materialize_episode_inputs( locale=locale, assets_dir=assets_dir, probe_mix=probe_mix, - tool_calling_themes=tool_calling_themes, toolset_seed_path=toolset_seed_path, match_persona_language=match_persona_language, drop_persona=False, diff --git a/src/usersim/cli/simulate.py b/src/usersim/cli/simulate.py index d59048ea..38260558 100644 --- a/src/usersim/cli/simulate.py +++ b/src/usersim/cli/simulate.py @@ -636,10 +636,7 @@ def _simulate_to_parquet( log_persona_language_population, persona_language_filter, ) - from usersim.cli._pipeline import ( - DEFAULT_TOOL_CALLING_THEMES, - build_simulator_config_builder, - ) + from usersim.cli._pipeline import build_simulator_config_builder from usersim.engine.core.locale import persona_dataset_locale from usersim.engine.core.manifest import write_simulator_manifest from usersim.engine.core.storage import ( @@ -797,7 +794,6 @@ def _simulate_to_parquet( sim_config=sim_cfg_for_manifest, locales_requested=run_locales, probe_mix=probe_mix, - tool_calling_themes=DEFAULT_TOOL_CALLING_THEMES, toolset_seed_path=toolset_seed_path, panel_path=Path(panel) if panel else None, assets_dir=Path(assets_dir).resolve() if assets_dir is not None else None, diff --git a/src/usersim/engine/core/manifest.py b/src/usersim/engine/core/manifest.py index f9eaac3b..3634e652 100644 --- a/src/usersim/engine/core/manifest.py +++ b/src/usersim/engine/core/manifest.py @@ -182,7 +182,6 @@ class ScopeInfo: locales: list[str] probes: list[str] probe_mix: dict[str, float] - tool_calling_themes: list[str] = field(default_factory=list) toolset_seed_path: str | None = None toolset_content_hash: str | None = None panel_path: str | None = None @@ -352,7 +351,6 @@ def _scope_from_dict(d: Mapping[str, Any]) -> ScopeInfo: locales=list(d.get("locales") or []), probes=list(d.get("probes") or []), probe_mix=dict(d.get("probe_mix") or {}), - tool_calling_themes=list(d.get("tool_calling_themes") or []), toolset_seed_path=d.get("toolset_seed_path"), toolset_content_hash=d.get("toolset_content_hash"), panel_path=d.get("panel_path"), @@ -724,7 +722,6 @@ def build_run_manifest( sim_config: Any, locales_requested: Sequence[str], probe_mix: Mapping[str, float], - tool_calling_themes: Sequence[str] = (), toolset_seed_path: Path | None = None, panel_path: Path | None = None, assets_dir: Path | None = None, @@ -789,7 +786,6 @@ def build_run_manifest( locales=sorted(set(list(locales_run) + list(locales_requested))), probes=list(aggregates["probes"]) or sorted(probe_mix.keys()), probe_mix=dict(probe_mix), - tool_calling_themes=list(tool_calling_themes), toolset_seed_path=str(toolset_seed_path) if toolset_seed_path else None, toolset_content_hash=_file_content_hash(toolset_seed_path) if toolset_seed_path else None, panel_path=str(panel_path) if panel_path else None, @@ -916,7 +912,6 @@ def write_simulator_manifest( sim_config: Any, locales_requested: Sequence[str], probe_mix: Mapping[str, float], - tool_calling_themes: Sequence[str] = (), toolset_seed_path: Path | None = None, panel_path: Path | None = None, assets_dir: Path | None = None, @@ -954,7 +949,6 @@ def write_simulator_manifest( sim_config=sim_config, locales_requested=locales_requested, probe_mix=probe_mix, - tool_calling_themes=tool_calling_themes, toolset_seed_path=toolset_seed_path, panel_path=panel_path, assets_dir=assets_dir, diff --git a/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md b/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md index f50aa10a..21e8bf40 100644 --- a/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md +++ b/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md @@ -120,9 +120,10 @@ toolsets_seed.parquet ── sample a toolset ──▶ assistant_model (model u - A call that breaks its tool's schema gets a `400` response instead of a simulated one, and a call without the user's key for a tool that takes one gets a `401` (see [Credentials](#credentials)). -- Conversation themes (some tool-expected, some not) come from - `DEFAULT_TOOL_CALLING_THEMES` in `cli/_pipeline.py`, exercising the - call/clarify/refuse decision. +- A conversation's theme is its toolset's category and name. A row whose + toolset has no `toolset_category` uses the theme the run drew from + [`en_US/themes.json`](en_US/themes.json) instead, which every locale reads + unless it has a `themes.json` of its own. ## Schema diff --git a/src/usersim/engine/probes/tool_calling/assets/tool_calling/en_US/themes.json b/src/usersim/engine/probes/tool_calling/assets/tool_calling/en_US/themes.json new file mode 100644 index 00000000..1520f5d4 --- /dev/null +++ b/src/usersim/engine/probes/tool_calling/assets/tool_calling/en_US/themes.json @@ -0,0 +1,42 @@ +[ + { + "type": "Weather & Location Lookup", + "description": "Look up current weather, forecasts, or location-based info.", + "tool_expected": true + }, + { + "type": "Calendar & Scheduling", + "description": "Create, modify, or check calendar events and availability.", + "tool_expected": true + }, + { + "type": "Data Retrieval & Search", + "description": "Search databases, look up records, or retrieve structured data.", + "tool_expected": true + }, + { + "type": "Financial Operations", + "description": "Check balances, make transactions, or calculate financial metrics.", + "tool_expected": true + }, + { + "type": "Communication & Messaging", + "description": "Send emails, messages, or notifications on behalf of the user.", + "tool_expected": true + }, + { + "type": "Multi-Step Workflow", + "description": "Complete a task requiring multiple sequential tool calls.", + "tool_expected": true + }, + { + "type": "General Knowledge Question", + "description": "Answer a factual question that requires no tools.", + "tool_expected": false + }, + { + "type": "Opinion or Advice Request", + "description": "Ask for subjective advice or opinions where tools are unnecessary.", + "tool_expected": false + } +] diff --git a/src/usersim/engine/probes/tool_calling/probe.py b/src/usersim/engine/probes/tool_calling/probe.py index fbe8b397..ecb4d91e 100644 --- a/src/usersim/engine/probes/tool_calling/probe.py +++ b/src/usersim/engine/probes/tool_calling/probe.py @@ -65,6 +65,7 @@ format_persona_for_prompt, format_tools_for_prompt, load_prompt_file, + load_seeds, make_failed, register_probe, render_prompt, @@ -282,12 +283,21 @@ class ToolCallingProbe(ToolExecutionMixin, ToolCallingMixin, BaseProbe): PreviewField("num_tools", "Tools available"), ) + @classmethod + def themes(cls, locale: str) -> list[dict[str, Any]]: + """The themes in ``locale``'s ``themes.json``, or in the ``en_US`` file when the locale has none.""" + return load_seeds(locale, cls.label, "themes") + @classmethod def asset_problems(cls) -> list[str]: - """The prompts file the prompt defaults would mask, and the toolset seed rows draw their tools from.""" + """The prompts file the prompt defaults would mask, the themes, and the toolset seed rows draw their tools from.""" problems = [] if not load_prompt_file(_SCENARIO): problems.append(f"{_SCENARIO}/prompts.yaml is missing or empty") + try: + cls.themes("en_US") + except FileNotFoundError as error: + problems.append(f"no themes: {error}") toolsets = cls.assets_dir() / "toolsets_seed.parquet" if not toolsets.exists(): problems.append(f"no toolset seed at {toolsets}, so every row would fail at construction") diff --git a/tests/engine/core/test_manifest.py b/tests/engine/core/test_manifest.py index 64470498..3d2ff89f 100644 --- a/tests/engine/core/test_manifest.py +++ b/tests/engine/core/test_manifest.py @@ -158,7 +158,6 @@ def test_manifest_to_dict_and_from_dict_roundtrip(self) -> None: locales=["en_US", "ja_JP"], probes=["tool_calling", "general_open_ended"], probe_mix={"tool_calling": 0.5, "general_open_ended": 0.5}, - tool_calling_themes=["weather", "calendar"], trajectories_requested={"en_US": 2, "ja_JP": 2}, trajectories_completed={"en_US": 2, "ja_JP": 2}, ), @@ -284,6 +283,16 @@ def test_loads_what_writer_wrote(self, tmp_path: Path) -> None: loaded = load_run_manifest("1700000000", tmp_path) assert loaded == m + def test_loads_a_manifest_with_a_scope_key_it_no_longer_reads(self, tmp_path: Path) -> None: + """A manifest whose scope holds a key the loader does not read, such as tool_calling_themes, still loads.""" + m = _minimal_manifest("1700000000") + write_run_manifest(m, tmp_path) + path = manifest_path(tmp_path, "1700000000") + payload = json.loads(path.read_text()) + payload["scope"]["tool_calling_themes"] = ['{"type": "Weather & Location Lookup"}'] + path.write_text(json.dumps(payload)) + assert load_run_manifest("1700000000", tmp_path) == m + def test_returns_none_for_run_without_manifest(self, tmp_path: Path) -> None: # No manifest written. Old-run handling depends on this contract. assert load_run_manifest("1700000000", tmp_path) is None @@ -557,7 +566,6 @@ def test_assembles_all_blocks(self) -> None: sim_config=_StubSimConfig(random_seed=42), locales_requested=["en_US"], probe_mix={"tool_calling": 1.0}, - tool_calling_themes=("weather",), assets_dir=Path("/repo/assets"), trajectories_requested={"en_US": 1}, trajectory_df=traj, From 6c329bb5bbed964d60265dd1569bb41b40b52d80 Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Wed, 4 Nov 2026 11:50:50 -0500 Subject: [PATCH 2/9] feat(engine): let a probe declare the seed table its rows sample BaseProbe.seed_dataset names a table in the probe's asset folder, .parquet or .jsonl, that a run attaches as its seed dataset, so each row gets the columns of one of the table's rows. tool_calling declares its toolset catalog, toolsets_seed, and reads its tools from the row's tools column unless the run sets tools_column, so the run setup names no probe: resolve_seed_dataset_for_mix finds the table of the probe in the mix that declares one, refuses a mix with two, and keeps --toolset-seed-path and toolset_seed_path= as the override for it. usersim smoke and assert_probe_conforms check that the declared table is there, the extension fixture declares one so make check-extensions resolves it from an installed package, and the tool-calling template declares one. architecture/probes.md, the catalog README, the engine README's config table and the --toolset-seed-path help describe it. Signed-off-by: Yev Meyer --- architecture/probes.md | 1 + docs/engine/README.md | 6 +- src/usersim/cli/_pipeline.py | 193 +++++++----------- src/usersim/cli/simulate.py | 24 +-- src/usersim/cli/smoke.py | 10 +- src/usersim/engine/core/probes.py | 43 ++++ .../assets/tool_calling/README.md | 13 +- .../engine/probes/tool_calling/probe.py | 32 +-- src/usersim/testing/__init__.py | 3 + templates/probe/generator_tool_calling.py | 22 +- .../fixtures/extension_package/pyproject.toml | 2 +- .../assets/fixture_probe/fixture_seed.jsonl | 1 + .../usersim_extension_fixture/probe.py | 1 + .../extension_package/verify_seams.py | 11 + tests/test_cli.py | 7 +- tests/test_probe_names_in_shared_code.py | 2 +- tests/test_simulator_pipeline_e2e.py | 41 ++-- tests/test_toolset_seed_cli.py | 91 +++++---- 18 files changed, 272 insertions(+), 231 deletions(-) create mode 100644 tests/fixtures/extension_package/usersim_extension_fixture/assets/fixture_probe/fixture_seed.jsonl diff --git a/architecture/probes.md b/architecture/probes.md index 07a78c28..2d143fe9 100644 --- a/architecture/probes.md +++ b/architecture/probes.md @@ -135,6 +135,7 @@ than into a branch on its label. | `authoring_guide` | `None` | the pre-flight, when a run asks for a locale the probe has no bank for | | `asset_problems` | no problems | `usersim smoke` and `assert_probe_conforms` | | `themes` | `None`: no theme | run setup, which draws one per row into the `theme` column | +| `seed_dataset` | `None`: no seed table | run setup, which attaches `.parquet` or `.jsonl` from the probe's asset folder and gives each row the columns of one of its rows; a mix with two such probes is refused; smoke and `assert_probe_conforms` check the table is there | | `model_aliases` | `()` | run setup, which adds each alias the models file defines to every row's models | | `output_columns` | `()` | the run's `side_effect_columns`; a column left out is dropped | | `supports_assistant_resampling` | `True`; `False` with the tool mixins | judge-gated assistant resampling (`max_assistant_attempts > 1`) | diff --git a/docs/engine/README.md b/docs/engine/README.md index c36930b9..e9224edb 100644 --- a/docs/engine/README.md +++ b/docs/engine/README.md @@ -454,10 +454,10 @@ diagnostics in the simulator-health report quantify that for a given run, and | `persona_column` | str | `"persona"` | Column containing persona dict | | `probe_type_column` | str | `"probe_type"` | Column carrying the probe label that the dispatcher resolves through `_PROBE_REGISTRY` | | `theme_column` | str | `"theme"` | Column with theme/topic | -| `tools_column` | str? | `None` | Tool definitions (tool_calling only) | -| `toolset_name_column` | str? | `None` | Toolset name (tool_calling only) | +| `tools_column` | str? | `None` | The column `tool_calling` reads its tools from; `None` reads `tools`, the column of its seed table | +| `toolset_name_column` | str? | `None` | Read by no probe; a stored config may set it | | `locale` | str | `"en_US"` | Locale for conversation language | -| `assets_dir` | str? | `None` | Optional override for the asset-bank root (defaults to repo `assets/`); per-probe env vars also work | +| `assets_dir` | str? | `None` | The asset root every probe is pinned to (`--assets-dir`); `None` reads each probe's own assets, after any `USERSIM_ASSET_PATH` or `USERSIM_ASSETS_DIR` override | | `max_tools` | int | `5` | Max tools sampled per row (tool_calling) | | `max_turns` | int | `5` | Max conversation turns | | `max_steps` | int | `10` | Max tool steps per turn | diff --git a/src/usersim/cli/_pipeline.py b/src/usersim/cli/_pipeline.py index 2798f687..3e098e42 100644 --- a/src/usersim/cli/_pipeline.py +++ b/src/usersim/cli/_pipeline.py @@ -88,77 +88,50 @@ def _probe_themes(probe: str, locale: str, assets_dir: Path | None) -> list[str] return [json.dumps(theme, ensure_ascii=False) for theme in themes] -#: Seed-file suffixes the ``tool_calling`` toolset seed may use. Both are -#: accepted by Data Designer's -#: ``LocalFileSeedSource`` (``VALID_DATASET_FILE_EXTENSIONS``); parquet is -#: what ships, jsonl is the hand-editable alternative. -TOOLSET_SEED_SUFFIXES: tuple[str, ...] = (".parquet", ".jsonl") - - -def resolve_toolset_seed_path( +def resolve_seed_dataset_path( + probe: str, assets_dir: Path | str | None = None, toolset_seed_path: Path | str | None = None, ) -> Path | None: - """The ``tool_calling`` toolset seed to use, or ``None`` if there isn't one. - - Two sources, at least one of which must be given: - - - ``toolset_seed_path`` — an explicit file, which wins when both are - passed. - It must exist: you named a path, and a run that silently ignores it - is worse than one that stops. - - ``assets_dir`` — the asset root to look under, accepting - ``toolsets_seed.parquet`` or ``toolsets_seed.jsonl`` - (``TOOLSET_SEED_SUFFIXES``). + """The seed table ``probe`` reads, or ``None`` if its asset folder holds none. - Lives here rather than in the CLI because ``assets_dir`` is already a - builder argument: any caller that names an asset root can have its - toolset found for it, including the notebooks. - Passing ``toolset_seed_path`` is an override, not a requirement. + ``probe`` must declare a ``seed_dataset``. Two sources: - Returns ``None`` only when ``assets_dir`` holds neither suffix. - Absence is a resolution result; whether it matters depends on the - probe mix, which ``_wire_toolsets`` decides. + - ``toolset_seed_path`` — an explicit file, which wins when given. It + must exist: you named a path, and a run that silently ignores it is + worse than one that stops. + - the probe's asset folder, under ``assets_dir`` when it is given and + along the asset search path otherwise, holding ``.parquet`` + or ``.jsonl`` (``SEED_DATASET_SUFFIXES``). - Raises ``ConfigError`` (a ``ValueError``) — never ``SystemExit`` — so + Raises ``ConfigError`` (a ``ValueError``), never ``SystemExit``, so library callers can handle it and ``cli.main`` can present it as a - message rather than a traceback, when an explicit path is unusable or an - asset root holds BOTH suffixes. That last is a configuration error rather + message rather than a traceback, when an explicit path is unusable or the + folder holds both suffixes. That last is a configuration error rather than a precedence question, since silently preferring one would make an - edit to the other look like it had no effect. Passing neither source - searches the configured asset path, which is how an unpinned run finds - the shipped seed. - - Routes through ``probe_assets_dir`` so the path is composed against the - same source-of-truth helper the asset loaders use, rather than a string - literal that can drift from the real layout. + edit to the other look like it had no effect. """ + from usersim.engine.core._assets import probe_assets_dir + from usersim.engine.core.probes import SEED_DATASET_SUFFIXES, resolve_probe if toolset_seed_path is not None: resolved = Path(toolset_seed_path).resolve() if not resolved.is_file(): - raise ConfigError(f"toolset seed not found: {resolved}") - if resolved.suffix not in TOOLSET_SEED_SUFFIXES: + raise ConfigError(f"seed table not found: {resolved}") + if resolved.suffix not in SEED_DATASET_SUFFIXES: raise ConfigError( - f"unsupported toolset seed suffix {resolved.suffix!r}: " - f"{resolved}. Expected one of {TOOLSET_SEED_SUFFIXES}." + f"unsupported seed table suffix {resolved.suffix!r}: {resolved}. Expected one of {SEED_DATASET_SUFFIXES}." ) return resolved - from usersim.engine.core._assets import probe_assets_dir - - # ``None`` means the caller did not pin a root, so the seed is looked - # for along the same search path the loaders use. - probe_dir = probe_assets_dir( - "tool_calling", - assets_root=Path(assets_dir).resolve() if assets_dir is not None else None, - ) - candidates = [probe_dir / f"toolsets_seed{sfx}" for sfx in TOOLSET_SEED_SUFFIXES] - present = [p for p in candidates if p.is_file()] + stem = resolve_probe(probe).seed_dataset + probe_dir = probe_assets_dir(probe, assets_root=Path(assets_dir).resolve() if assets_dir is not None else None) + present = [probe_dir / f"{stem}{suffix}" for suffix in SEED_DATASET_SUFFIXES] + present = [path for path in present if path.is_file()] if len(present) > 1: raise ConfigError( - "ambiguous tool_calling toolset seed: " - + ", ".join(p.name for p in present) + f"ambiguous {probe} seed table: " + + ", ".join(path.name for path in present) + f" both exist in {probe_dir}. Keep exactly one." ) return present[0] if present else None @@ -298,80 +271,62 @@ def _runnable_locales(runnable: Sequence[str], bank_locale: Callable[[str], str] return described -def resolve_toolset_seed_for_mix( - probe_mix: dict[str, float], +def seed_dataset_probe(probe_mix: Mapping[str, float]) -> str | None: + """The probe in ``probe_mix`` that reads a seed table, or ``None`` when none does. + + A run attaches one seed table, so a mix with two such probes is a + ``ConfigError``: run them separately. + """ + from usersim.engine.core.probes import resolve_probe + + readers = [probe for probe in probe_mix if resolve_probe(probe).seed_dataset is not None] + if len(readers) > 1: + raise ConfigError( + f"{', '.join(readers)} each read a seed table, and a run attaches one. Run them in separate runs." + ) + return readers[0] if readers else None + + +def resolve_seed_dataset_for_mix( + probe_mix: Mapping[str, float], assets_dir: Path | str | None = None, toolset_seed_path: Path | str | None = None, ) -> Path | None: - """Resolve a toolset only when the run actually includes ``tool_calling``. + """The seed table a run of ``probe_mix`` attaches, or ``None`` when no probe in it reads one. - A non-tool run must not be blocked by an ambiguous or missing toolset in - an otherwise valid asset tree, and its manifest must not claim it used a - file that never entered the pipeline. A tool run must resolve a real seed. - Conversely, an explicit override on a non-tool run is rejected rather than - silently ignored. + A run without such a probe is not blocked by an ambiguous or missing + table in an otherwise valid asset tree, and its manifest does not claim + it used a file that never entered the pipeline; an explicit + ``toolset_seed_path`` for it is refused rather than silently ignored. A + run with one must resolve a real table. """ - if "tool_calling" not in probe_mix: + probe = seed_dataset_probe(probe_mix) + if probe is None: if toolset_seed_path is not None: raise ConfigError( - "toolset_seed_path was provided, but probe_mix does not " - "include tool_calling. Add tool_calling or remove the override." + "toolset_seed_path was provided, but no probe in probe_mix reads a seed table. " + "Add the probe it is for, or remove the override." ) return None - resolved = resolve_toolset_seed_path(assets_dir, toolset_seed_path) + resolved = resolve_seed_dataset_path(probe, assets_dir, toolset_seed_path) if resolved is None: + from usersim.engine.core.probes import resolve_probe + + stem = resolve_probe(probe).seed_dataset raise ConfigError( - "probe_mix includes tool_calling but no toolset seed is available. " - "Pass toolset_seed_path=, point assets_dir at a tree with " - "tool_calling/toolsets_seed.{parquet,jsonl}, or drop tool_calling " - "from the mix." + f"probe_mix includes {probe}, which reads the seed table {stem}.parquet or {stem}.jsonl, and none is " + f"available. Pass toolset_seed_path=, point assets_dir at a tree with {probe}/{stem}.parquet or " + f"{probe}/{stem}.jsonl, or drop {probe} from the mix." ) return resolved -def _wire_toolsets( - config_builder: Any, - dd: Any, - toolset_seed_path: Path | None, -) -> dict[str, Any]: - """Attach ``tool_calling``'s toolset seed and return its config keys. - - Both halves of the wiring — the builder mutation and the - ``ConversationSimulatorConfig`` keys — are decided here together, so the - two cannot drift apart: a run that attaches the seed dataset always gets - the matching tool columns, and one that does not always leaves them - unset. - - Called only from the ``tool_calling`` branch. The mix-aware resolver - normally rejects a missing file first; the check here is a defensive - invariant. Failing at config-build time rather than letting the run start - is the point: with the columns unset, every ``tool_calling`` row aborts at - probe construction, so the whole partition comes back failed and the - cause is visible only by reading ``simulation_outcome`` afterwards. - - ``ConfigError`` matches this module's other user-fixable errors (see - the ``probe_mix`` validation above) and guards every caller — the CLI, - and the notebooks — rather than just the one that - happens to resolve the path. ``cli.main`` catches it and prints the - message alone, so this text is the entire error output a user sees. - - Takes an already-resolved path: the caller resolves only after confirming - that ``tool_calling`` is in the mix, so this function needs no - ``assets_dir``. - """ - if toolset_seed_path is None or not Path(toolset_seed_path).is_file(): - raise ConfigError( - "probe_mix includes tool_calling but no toolset seed is " - f"available (resolved to {toolset_seed_path!r}). Pass " - "toolset_seed_path=, point assets_dir at a tree with " - "tool_calling/toolsets_seed.{parquet,jsonl}, or drop " - "tool_calling from the mix." - ) +def _wire_seed_dataset(config_builder: Any, dd: Any, seed_path: Path) -> None: + """Attach ``seed_path`` as the run's seed dataset, one shuffled row per row the run makes.""" config_builder.with_seed_dataset( - dd.LocalFileSeedSource(path=str(toolset_seed_path)), + dd.LocalFileSeedSource(path=str(seed_path)), sampling_strategy=dd.SamplingStrategy.SHUFFLE, ) - return {"tools_column": "tools", "toolset_name_column": "toolset_name"} def _set_run_config(data_designer, dd) -> None: @@ -457,7 +412,7 @@ def build_simulator_config_builder( data_designer = DataDesigner(**dd_kwargs) _set_run_config(data_designer, dd) config_builder = dd.DataDesignerConfigBuilder(model_configs=to_model_configs(models)) - toolset_kwargs = _add_episode_input_columns( + _add_episode_input_columns( config_builder, dd, locale=locale, @@ -474,7 +429,6 @@ def build_simulator_config_builder( persona_column="persona", probe_type_column="probe_type", theme_column="theme", - **toolset_kwargs, locale=locale, assets_dir=_recorded_assets_dir(assets_dir), max_turns=max_turns, @@ -504,15 +458,15 @@ def _add_episode_input_columns( toolset_seed_path: Path | None, match_persona_language: bool, drop_persona: bool, -) -> dict[str, Any]: - """Install the one canonical persona/probe/theme/toolset sampler graph.""" +) -> None: + """Install the one canonical persona, probe, theme and seed-table sampler graph.""" from usersim.engine.core.locale import persona_dataset_locale persona_locale = persona_dataset_locale(locale, datasets_dir=DEFAULT_PERSONA_DATASETS_DIR) unknown = [key for key in probe_mix if key not in set(known_probes())] if unknown: raise ConfigError(f"Unknown probe_type(s) in probe_mix: {unknown}. Registered: {sorted(known_probes())}.") - resolved_toolset_seed = resolve_toolset_seed_for_mix(probe_mix, assets_dir, toolset_seed_path) + seed_path = resolve_seed_dataset_for_mix(probe_mix, assets_dir, toolset_seed_path) config_builder.add_column( dd.SamplerColumnConfig( name="persona", @@ -534,12 +488,9 @@ def _add_episode_input_columns( params=dd.CategorySamplerParams(values=list(probe_mix), weights=list(probe_mix.values())), ) ) - themes: dict[str, list[str]] = {} - toolset_kwargs: dict[str, Any] = {} - for probe in probe_mix: - themes[probe] = _probe_themes(probe, locale, assets_dir) - if probe == "tool_calling": - toolset_kwargs = _wire_toolsets(config_builder, dd, resolved_toolset_seed) + themes = {probe: _probe_themes(probe, locale, assets_dir) for probe in probe_mix} + if seed_path is not None: + _wire_seed_dataset(config_builder, dd, seed_path) config_builder.add_column( dd.SamplerColumnConfig( name="theme", @@ -547,7 +498,6 @@ def _add_episode_input_columns( params=dd.SubcategorySamplerParams(category="probe_type", values=themes), ) ) - return toolset_kwargs def materialize_episode_inputs( @@ -576,7 +526,7 @@ def materialize_episode_inputs( designer = DataDesigner(**to_data_designer_kwargs(models)) _set_run_config(designer, dd) builder = dd.DataDesignerConfigBuilder() - toolset_kwargs = _add_episode_input_columns( + _add_episode_input_columns( builder, dd, locale=locale, @@ -597,7 +547,6 @@ def materialize_episode_inputs( replay_assistant_reasoning=assistant_replays_reasoning(models, store_reasoning=store_reasoning), finance_tier_mix=finance_tier_mix, finance_retrieval_mode=finance_retrieval_mode, - **toolset_kwargs, ) model_identities = {spec.alias: SimpleNamespace(model_name=spec.model) for spec in models.models} sampled = designer.preview(builder, num_records=num_rows).dataset diff --git a/src/usersim/cli/simulate.py b/src/usersim/cli/simulate.py index 38260558..14b14fdc 100644 --- a/src/usersim/cli/simulate.py +++ b/src/usersim/cli/simulate.py @@ -43,7 +43,7 @@ from usersim.cli._errors import ConfigError from usersim.cli._persona_language import persona_dataset_report -from usersim.cli._pipeline import resolve_toolset_seed_for_mix, verify_probe_assets, verify_probe_locales +from usersim.cli._pipeline import resolve_seed_dataset_for_mix, verify_probe_assets, verify_probe_locales logger = logging.getLogger("usersim.cli.simulate") @@ -147,14 +147,14 @@ def register(subparsers: argparse._SubParsersAction) -> None: type=Path, default=None, help=( - "Toolset file for the tool_calling probe (.parquet or .jsonl), " - "one row per toolset with columns toolset_source, " - "toolset_category, toolset_name, num_tools and tools (JSON list " - "of OpenAI-style function specs) -- see the Schema section of " + "Seed table (.parquet or .jsonl) for the probe in --probe-mix that " + "reads one, in place of the table it declares in its assets. For " + "tool_calling, its toolset catalog: one row per toolset with columns " + "toolset_source, toolset_category, toolset_name, num_tools and tools " + "(JSON list of OpenAI-style function specs) -- see the Schema section of " "engine/probes/tool_calling/assets/tool_calling/README.md in the package. " - "Defaults to toolsets_seed.{parquet,jsonl} in the tool_calling assets, under --assets-dir when given. " - "Requires tool_calling in --probe-mix; a missing or unsupported " - "explicit file is an error." + "Requires a probe in --probe-mix that reads a seed table; a missing or " + "unsupported explicit file is an error." ), ) p.add_argument( @@ -377,7 +377,7 @@ def run(args: argparse.Namespace) -> int: # Resolved before anything is printed so a bad config fails cleanly # instead of interleaving with a half-written plan. verify_probe_assets(probe_mix, run_locales, args.assets_dir) - seed = resolve_toolset_seed_for_mix( + seed = resolve_seed_dataset_for_mix( probe_mix, args.assets_dir, args.toolset_seed_path, @@ -670,10 +670,10 @@ def _simulate_to_parquet( len(already_done), ) - # Resolve only for a run that includes tool_calling. The builder also + # Resolve only for a run whose mix reads a seed table. The builder also # enforces this for library callers, while doing it here gives the manifest - # the exact file used and keeps non-tool runs independent of toolset assets. - toolset_seed_path = resolve_toolset_seed_for_mix( + # the exact file used and keeps other runs independent of seed tables. + toolset_seed_path = resolve_seed_dataset_for_mix( probe_mix, assets_dir, toolset_seed_path, diff --git a/src/usersim/cli/smoke.py b/src/usersim/cli/smoke.py index 393e62e7..3c72ed28 100644 --- a/src/usersim/cli/smoke.py +++ b/src/usersim/cli/smoke.py @@ -438,7 +438,7 @@ def _check_probe_registry(report: SmokeReport) -> None: def _check_probe_assets(report: SmokeReport) -> None: - """Run every registered probe's ``asset_problems`` check. + """Run every registered probe's ``asset_problems`` check, and check the seed table it declares. Each probe checks the assets it ships, such as a bank per locale, so a missing or broken bank fails at install time rather than partway through @@ -447,7 +447,7 @@ def _check_probe_assets(report: SmokeReport) -> None: package is checked the same way as a built-in one. """ try: - from usersim.engine.core.probes import BaseProbe, known_probes, resolve_probe + from usersim.engine.core.probes import BaseProbe, known_probes, resolve_probe, seed_dataset_problems labels = known_probes() except Exception as e: @@ -460,10 +460,12 @@ def _check_probe_assets(report: SmokeReport) -> None: for label in labels: try: cls = resolve_probe(label) - if inspect.getattr_static(cls, "asset_problems") is default_check: + own_check = inspect.getattr_static(cls, "asset_problems") is not default_check + if not own_check and cls.seed_dataset is None: continue checked.append(label) - problems.extend(f"{label}: {problem}" for problem in cls.asset_problems()) + found = [*(cls.asset_problems() if own_check else []), *seed_dataset_problems(cls)] + problems.extend(f"{label}: {problem}" for problem in found) except Exception as e: # smoke reports, never raises problems.append(f"{label}: asset_problems raised {type(e).__name__}: {e}") diff --git a/src/usersim/engine/core/probes.py b/src/usersim/engine/core/probes.py index 656e2e0b..e1c4d619 100644 --- a/src/usersim/engine/core/probes.py +++ b/src/usersim/engine/core/probes.py @@ -246,6 +246,41 @@ class PreviewField: from_metadata: bool = False +#: The suffixes a probe's ``seed_dataset`` table may have; both are files Data +#: Designer's ``LocalFileSeedSource`` reads. A folder holding both for one +#: table is refused, so an edit to the one a run ignores cannot look like an +#: edit that had no effect. +SEED_DATASET_SUFFIXES: tuple[str, ...] = (".parquet", ".jsonl") + + +def seed_dataset_problems(cls: type) -> list[str]: + """What is wrong with the seed table ``cls.seed_dataset`` names; empty when it names none. + + The name must be a file name without its suffix, and the probe's asset + folder must hold exactly one of ``.parquet`` and ``.jsonl``. + """ + seed = getattr(cls, "seed_dataset", None) + if seed is None: + return [] + if not (isinstance(seed, str) and seed and Path(seed).name == seed and not Path(seed).suffix): + return [ + f"{cls.__qualname__}.seed_dataset must be None or a file name without its suffix, such as " + f"'toolsets_seed'; got {seed!r}" + ] + # _assets imports this module at its top, so importing it here at the top would be circular. + from usersim.engine.core._assets import probe_assets_dir + + folder = probe_assets_dir(cls.label) + names = [f"{seed}{suffix}" for suffix in SEED_DATASET_SUFFIXES] + present = [name for name in names if (folder / name).is_file()] + if len(present) != 1: + return [ + f"{cls.__qualname__}.seed_dataset is {seed!r}, and {folder} holds " + f"{'neither' if not present else 'both'} of {', '.join(names)}; it must hold one." + ] + return [] + + class BaseProbe: """Concrete probe with sensible defaults for every optional hook. @@ -327,6 +362,14 @@ def assets_dir(cls) -> Path: # pre-flight links it when a run asks for a locale the probe has no bank for. authoring_guide: str | None = None + # The name, without its suffix, of a table in this probe's asset folder, + # ``.parquet`` or ``.jsonl`` (``SEED_DATASET_SUFFIXES``), that a + # run samples one row of for every row it makes, so each column of the + # table is a column of the row the probe reads. A run attaches one such + # table, so a probe mix with two probes that declare one is refused before + # any model call. ``None`` reads no table. + seed_dataset: str | None = None + @classmethod def bank_locale(cls, locale: str, probe_dir: Path) -> str: """The locale whose bank in ``probe_dir`` a row in ``locale`` runs on. diff --git a/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md b/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md index 21e8bf40..19d04ddf 100644 --- a/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md +++ b/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md @@ -170,10 +170,15 @@ call. ### Using your own catalog -A run reads `/tool_calling/toolsets_seed.parquet` by default, or -`toolsets_seed.jsonl` if that is what is present. Keep exactly one: both -suffixes in the same directory is an error rather than a precedence rule, -so that editing the one the run ignores cannot look like an edit that had no effect. +The probe declares the catalog as its seed table (`seed_dataset = +"toolsets_seed"`), so a run reads `toolsets_seed.parquet` from the probe's +asset folder, or `toolsets_seed.jsonl` if that is what is present, and gives +each row the columns of one catalog row. Under `--assets-dir`, the folder is +`/tool_calling/`. Keep exactly one: both suffixes in the same +directory is an error rather than a precedence rule, so that editing the one +the run ignores cannot look like an edit that had no effect. A run holds one +seed table, so `tool_calling` cannot share a run with another probe that +reads one. To point a run at a file somewhere else, without touching this tree: diff --git a/src/usersim/engine/probes/tool_calling/probe.py b/src/usersim/engine/probes/tool_calling/probe.py index ecb4d91e..87470ffd 100644 --- a/src/usersim/engine/probes/tool_calling/probe.py +++ b/src/usersim/engine/probes/tool_calling/probe.py @@ -117,6 +117,11 @@ def _build_tool_context(tool_subset: list) -> str: return "\n".join(lines) if lines else "No tools available." +def _tools_column(cfg: Any) -> str: + """The row column holding the toolset's tools: the run's ``tools_column``, else the catalog's ``tools``.""" + return getattr(cfg, "tools_column", None) or "tools" + + def _row_identity(data: dict) -> str: """The row's identity, which seeds its tool draw and the credentials its user holds. @@ -270,6 +275,9 @@ class ToolCallingProbe(ToolExecutionMixin, ToolCallingMixin, BaseProbe): label = "tool_calling" assets_root = Path(__file__).parent / "assets" + # The toolset catalog: each row draws one toolset and reads its tools from + # the ``tools`` column. + seed_dataset = "toolsets_seed" description = ( "Tests the assistant's ability to invoke tools correctly in response to user " "requests, including argument shaping, schema validation, and recovery from " @@ -290,7 +298,7 @@ def themes(cls, locale: str) -> list[dict[str, Any]]: @classmethod def asset_problems(cls) -> list[str]: - """The prompts file the prompt defaults would mask, the themes, and the toolset seed rows draw their tools from.""" + """The prompts file the prompt defaults would mask, and the themes.""" problems = [] if not load_prompt_file(_SCENARIO): problems.append(f"{_SCENARIO}/prompts.yaml is missing or empty") @@ -298,20 +306,16 @@ def asset_problems(cls) -> list[str]: cls.themes("en_US") except FileNotFoundError as error: problems.append(f"no themes: {error}") - toolsets = cls.assets_dir() / "toolsets_seed.parquet" - if not toolsets.exists(): - problems.append(f"no toolset seed at {toolsets}, so every row would fail at construction") return problems def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) cfg = self._cfg - if not getattr(cfg, "tools_column", None): - raise ValueError("tool_calling: cfg.tools_column not configured") - if cfg.tools_column not in self._data: - raise ValueError(f"tool_calling: column {cfg.tools_column!r} missing in data") + tools_column = _tools_column(cfg) + if tools_column not in self._data: + raise ValueError(f"tool_calling: column {tools_column!r} missing in data") - tools_raw = self._data[cfg.tools_column] + tools_raw = self._data[tools_column] all_tools = _normalize_tool_list(tools_raw) # Drawn from the row's own identity rather than the process RNG. The # subset shapes the entire conversation but is not part of the @@ -513,12 +517,10 @@ async def simulate_tool_calling( tests that import the symbol. Returns a structured failure record on missing-tools-column rather than raising. """ - if not getattr(cfg, "tools_column", None): - logger.warning(" |-- tool_calling: tools_column not configured") - return make_failed("tools_column not configured") - if cfg.tools_column not in data: - logger.warning(f" |-- tool_calling: column '{cfg.tools_column}' not found in data") - return make_failed(f"tools_column '{cfg.tools_column}' missing in data") + tools_column = _tools_column(cfg) + if tools_column not in data: + logger.warning(f" |-- tool_calling: column '{tools_column}' not found in data") + return make_failed(f"tools_column '{tools_column}' missing in data") probe = ToolCallingProbe( persona=persona, diff --git a/src/usersim/testing/__init__.py b/src/usersim/testing/__init__.py index 6fcf942e..08db5500 100644 --- a/src/usersim/testing/__init__.py +++ b/src/usersim/testing/__init__.py @@ -96,6 +96,7 @@ def probe_conformance_problems(probe: str | type) -> list[str]: BaseProbe, PreviewField, known_probes, + seed_dataset_problems, ) problems: list[str] = [] @@ -220,6 +221,8 @@ def probe_conformance_problems(probe: str | type) -> list[str]: f"installed package; got {guide!r}" ) + problems.extend(seed_dataset_problems(cls)) + for klass, key in _reserved_metadata_writes(cls): problems.append( f"{klass} writes state.metadata[{key!r}], which the conversation loop owns. " diff --git a/templates/probe/generator_tool_calling.py b/templates/probe/generator_tool_calling.py index f74ffa33..05f20a2c 100644 --- a/templates/probe/generator_tool_calling.py +++ b/templates/probe/generator_tool_calling.py @@ -20,9 +20,11 @@ ``after_assistant_turn``.) What you provide: -- ``__init__(...)`` — sample tools from ``self._data[cfg.tools_column]`` - (typically a CLI-provided tool registry), validate, store on - ``self``. +- ``seed_dataset`` — the name of the tool table in your asset folder + (``.parquet`` or ``.jsonl``); a run gives each row the + columns of one of its rows, and ``--toolset-seed-path`` swaps in another. +- ``__init__(...)`` — sample tools from the row's ``tools`` column, + validate, store on ``self``. - ``get_user_system_prompt`` — typical persona-grounded user prompt. - ``get_tools_for_assistant`` — returns the sampled OpenAI tool specs. - ``after_assistant_turn`` — intercepts tool calls, executes them @@ -58,15 +60,17 @@ class DemoToolCallingProbe(ToolCallingMixin, BaseProbe): """Tool-calling capability probe with mid-loop tool execution.""" label = "demo_tool_calling" + # assets/demo_tool_calling/toolsets.parquet (or .jsonl): one row per + # toolset, with a ``tools`` column of OpenAI-style function specs. + seed_dataset = "toolsets" def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - cfg = self._cfg - if not getattr(cfg, "tools_column", None): - raise ValueError("demo_tool_calling: cfg.tools_column not configured") - # Real probe: parse self._data[cfg.tools_column], validate, - # sample subset. See tool_calling/probe.py for the - # canonical tool-sampling logic. + if "tools" not in self._data: + raise ValueError("demo_tool_calling: the row has no tools column; is the seed table attached?") + # Real probe: parse self._data["tools"], validate, sample a + # subset. See tool_calling/probe.py for the canonical + # tool-sampling logic. self._tools: list[dict] = [] # populated from row def get_user_system_prompt(self) -> str: diff --git a/tests/fixtures/extension_package/pyproject.toml b/tests/fixtures/extension_package/pyproject.toml index f3e92ee5..63ba43d2 100644 --- a/tests/fixtures/extension_package/pyproject.toml +++ b/tests/fixtures/extension_package/pyproject.toml @@ -33,4 +33,4 @@ fixture-cmd = "usersim_extension_fixture.command:register" include = ["usersim_extension_fixture*"] [tool.setuptools.package-data] -usersim_extension_fixture = ["assets/**/*.yaml"] +usersim_extension_fixture = ["assets/**/*.yaml", "assets/**/*.jsonl"] diff --git a/tests/fixtures/extension_package/usersim_extension_fixture/assets/fixture_probe/fixture_seed.jsonl b/tests/fixtures/extension_package/usersim_extension_fixture/assets/fixture_probe/fixture_seed.jsonl new file mode 100644 index 00000000..fd297d35 --- /dev/null +++ b/tests/fixtures/extension_package/usersim_extension_fixture/assets/fixture_probe/fixture_seed.jsonl @@ -0,0 +1 @@ +{"fixture_seed_value": "seeded"} diff --git a/tests/fixtures/extension_package/usersim_extension_fixture/probe.py b/tests/fixtures/extension_package/usersim_extension_fixture/probe.py index e4f472ab..85553c9a 100644 --- a/tests/fixtures/extension_package/usersim_extension_fixture/probe.py +++ b/tests/fixtures/extension_package/usersim_extension_fixture/probe.py @@ -22,6 +22,7 @@ class FixtureProbe(BaseProbe): default_scorers = ("fixture_scorer",) locale_bank_check = staticmethod(_check_locale_bank) authoring_guide = "https://example.com/fixture-probe/locales" + seed_dataset = "fixture_seed" @classmethod def model_aliases(cls, cfg) -> tuple[str, ...]: diff --git a/tests/fixtures/extension_package/verify_seams.py b/tests/fixtures/extension_package/verify_seams.py index e7731a02..de7f7127 100644 --- a/tests/fixtures/extension_package/verify_seams.py +++ b/tests/fixtures/extension_package/verify_seams.py @@ -138,6 +138,17 @@ def preflight_refusal(locales: list[str]) -> str: str(themes), ) +# A contributed probe's seed table resolves from its installed assets, as a run +# that holds the probe attaches it. +from usersim.cli._pipeline import resolve_seed_dataset_for_mix # noqa: E402 + +seed = resolve_seed_dataset_for_mix({"fixture_probe": 1.0}) +check( + "seed_dataset resolves a contributed seed table", + seed is not None and seed.name == "fixture_seed.jsonl" and "usersim_extension_fixture" in seed.parts, + str(seed), +) + # A model a contributed probe declares joins the models every row resolves. from usersim.engine.core.probes import declared_model_aliases # noqa: E402 diff --git a/tests/test_cli.py b/tests/test_cli.py index 7c30cda4..b78de531 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1232,10 +1232,11 @@ def test_missing_toolset_seed_raises_config_error(self, tmp_path): """The real path: an explicit ``--toolset-seed-path`` that is not there is a user error, not a crash.""" from usersim.cli._errors import ConfigError - from usersim.cli._pipeline import resolve_toolset_seed_path + from usersim.cli._pipeline import resolve_seed_dataset_path - with pytest.raises(ConfigError, match="toolset seed not found"): - resolve_toolset_seed_path( + with pytest.raises(ConfigError, match="seed table not found"): + resolve_seed_dataset_path( + "tool_calling", toolset_seed_path=tmp_path / "absent.parquet", ) diff --git a/tests/test_probe_names_in_shared_code.py b/tests/test_probe_names_in_shared_code.py index bcaf21cf..83f7fa0d 100644 --- a/tests/test_probe_names_in_shared_code.py +++ b/tests/test_probe_names_in_shared_code.py @@ -28,7 +28,7 @@ class instead (``architecture/probes.md``, "What a probe declares"). #: How many built-in probe labels each file outside ``engine/probes/`` may #: write as strings, by path under ``src/usersim/``. _ALLOWED: dict[str, int] = { - "cli/_pipeline.py": 7, + "cli/_pipeline.py": 4, "cli/smoke.py": 2, "engine/core/preview.py": 3, "engine/evaluator/axes.py": 6, diff --git a/tests/test_simulator_pipeline_e2e.py b/tests/test_simulator_pipeline_e2e.py index d7a2d7d0..006a3c56 100644 --- a/tests/test_simulator_pipeline_e2e.py +++ b/tests/test_simulator_pipeline_e2e.py @@ -4,21 +4,19 @@ """End-to-end wiring test: every shipped probe survives construction + single-turn dispatch when driven through ``cli/_pipeline.build_simulator_config_builder``. -This is the test that would have caught the -``cfg.tools_column not configured`` regression where every -``tool_calling`` trajectory failed at probe construction. The -existing per-probe tests (``tests/probes/test_*.py``) all use -synthetic ``ConversationSimulatorConfig`` fixtures that hard-code the -correct ``tools_column`` / ``persona_column`` / etc. values; they -therefore never exercise the wiring chain +A probe whose construction needs a setting or a column the run's wiring +does not provide fails here, at PR time, rather than on every row of a +live run. The per-probe tests (``tests/probes/