diff --git a/architecture/evaluator.md b/architecture/evaluator.md index 12f1034..50cebbf 100644 --- a/architecture/evaluator.md +++ b/architecture/evaluator.md @@ -44,7 +44,9 @@ under; `auto` combines with scorer names. Over a run holding every built-in probe it runs every scorer the capability report reads. A capability whose scorer was not dispatched renders as *untested* rather than as a failure, and `capabilities_missing_scorer()` exists so a run can warn about that instead -of silently producing an empty column. +of silently producing an empty column. The evaluator calls every scorer it runs +on every row, so a probe's scorer skips the rows of other probes, and the +capability report counts a skipped row as not applicable. ## The evaluation cell diff --git a/architecture/probes.md b/architecture/probes.md index 49dd211..f920918 100644 --- a/architecture/probes.md +++ b/architecture/probes.md @@ -135,7 +135,7 @@ than into a branch on its label. | `authoring_guide` | `None` | the pre-flight, when a run asks for a locale the probe has no bank for | | `asset_problems` | no problems | `usersim smoke` and `assert_probe_conforms` | | `themes` | `None`: no theme | run setup, which draws one per row into the `theme` column | -| `seed_dataset` | `None`: no seed table | run setup, which attaches `.parquet` or `.jsonl` from the probe's asset folder and gives each row the columns of one of its rows; a mix with two such probes is refused; smoke and `assert_probe_conforms` check the table is there | +| `seed_dataset` | `None`: no seed table | run setup, which attaches `.parquet` or `.jsonl` from the probe's asset folder and gives each of the probe's rows the columns of one of its rows, and the mix's other probes' rows those columns empty; a mix with two such probes is refused; smoke and `assert_probe_conforms` check the table is there | | `model_aliases` | `()` | run setup, which adds each alias the models file defines to every row's models | | `output_columns` | `()` | the run's `side_effect_columns`; a column left out is dropped | | `supports_assistant_resampling` | `True`; `False` with the tool mixins | judge-gated assistant resampling (`max_assistant_attempts > 1`) | diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md index 499ba75..a5b658c 100644 --- a/docs/engine/AUTHORING_A_PROBE.md +++ b/docs/engine/AUTHORING_A_PROBE.md @@ -1282,7 +1282,11 @@ rows, beside the rule-based `language_compliance`, `refusal_basics` and reports a name in `default_scorers` that no registered scorer holds. The evaluator calls each scorer it runs on every row, so return early for rows from other probes, for example when your probe's join key is missing, as the -`safety_agentic` scorer does with `action_request_id`. +`safety_agentic` scorer does with `action_request_id`. Return an empty mapping, +or an `error` that starts with `no ` and contains `scorer skipped`: the +capability report counts the row as not applicable rather than as evidence. +[`tests/engine/evaluator/test_scorers_skip_other_probes.py`](../../tests/engine/evaluator/test_scorers_skip_other_probes.py) +checks that every built-in probe's scorer skips another probe's row. A scorer in your probe's package may write its label; [`tests/test_probe_names_in_shared_code.py`](../../tests/test_probe_names_in_shared_code.py) diff --git a/docs/engine/README.md b/docs/engine/README.md index e9224ed..c639038 100644 --- a/docs/engine/README.md +++ b/docs/engine/README.md @@ -431,7 +431,8 @@ hook), see the source-of-truth docstrings in **Assistant evaluation is configured separately** via `TrajectoryEvaluatorConfig` (see Trajectory Evaluator section). Add new axes to `evaluator/axes.py::PROBE_SCORES` if needed; add a -scorer under `evaluator/scorers/your_probe.py` and include it in +scorer as `scorer.py` in the probe's package +(`engine/probes/your_probe/scorer.py`) and include its module path in `DEFAULT_SCORER_MODULES` to register it on import; name it in `--scorers` to run it, or list it in the probe's `default_scorers` so `--scorers auto` runs it. diff --git a/notebooks/01_simulate.ipynb b/notebooks/01_simulate.ipynb index 90c02b0..e0c502f 100644 --- a/notebooks/01_simulate.ipynb +++ b/notebooks/01_simulate.ipynb @@ -94,7 +94,12 @@ ")\n", "from usersim.cli._env import describe_loaded_env, load_local_env\n", "from usersim.cli._ngc import ensure_ngc_cli, ensure_ngc_org, ngc_available\n", - "from usersim.cli._pipeline import build_simulator_config_builder, known_probes, sample_trajectories\n", + "from usersim.cli._pipeline import (\n", + " build_simulator_config_builder,\n", + " clear_seed_columns_of_other_probes,\n", + " known_probes,\n", + " sample_trajectories,\n", + ")\n", "\n", "# --- Engine (asset + storage + preview helpers) ---\n", "from usersim.engine.core._assets import probe_assets_dir\n", @@ -704,6 +709,11 @@ " with tempfile.TemporaryDirectory(prefix=\"usersim_simulate_\") as _artifacts:\n", " result = data_designer.create(builder, num_records=NUM_ROWS_PER_LOCALE, artifact_path=_artifacts)\n", " df = result.load_dataset()\n", + " # Data Designer gives every row the seed table's columns; keep them to the\n", + " # probe that reads the table.\n", + " df = clear_seed_columns_of_other_probes(\n", + " df, probe_mix=PROBE_MIX, assets_dir=ASSETS_DIR, toolset_seed_path=_seed_path\n", + " )\n", " # Atomic per-locale write: this locale's partition lands on disk\n", " # before the next locale starts. A crash mid-run leaves earlier\n", " # locales' partitions intact and resumable. It holds the run lock, skips\n", diff --git a/src/usersim/cli/_pipeline.py b/src/usersim/cli/_pipeline.py index 3e098e4..159a723 100644 --- a/src/usersim/cli/_pipeline.py +++ b/src/usersim/cli/_pipeline.py @@ -321,6 +321,53 @@ def resolve_seed_dataset_for_mix( return resolved +def seed_dataset_columns(seed_path: Path) -> list[str]: + """The columns of the seed table at ``seed_path``: a parquet file's schema, or every key of a JSONL file.""" + if seed_path.suffix == ".parquet": + import pyarrow.parquet as pq + + return list(pq.read_schema(seed_path).names) + columns: dict[str, None] = {} + with seed_path.open(encoding="utf-8") as lines: + for line in lines: + if line.strip(): + columns.update(dict.fromkeys(json.loads(line))) + return list(columns) + + +def clear_seed_columns_of_other_probes( + df: Any, + *, + probe_mix: Mapping[str, float], + assets_dir: Path | str | None = None, + toolset_seed_path: Path | str | None = None, +) -> Any: + """``df`` with the seed table's columns emptied on every row of a probe that does not read the table. + + Data Designer gives each row of a run the columns of one seed-table row, + whatever the row's probe, so without this a row of another probe in the + mix would record the seed row it drew: a toolset its conversation never + offered, say. The table is the one the run attaches, resolved from the + same arguments, and ``df`` comes back unchanged when no probe in the mix + reads one. + """ + seed_path = resolve_seed_dataset_for_mix(probe_mix, assets_dir, toolset_seed_path) + reader = seed_dataset_probe(probe_mix) + if seed_path is None or reader is None or "probe_type" not in df.columns: + return df + columns = [column for column in seed_dataset_columns(seed_path) if column in df.columns] + others = df["probe_type"].astype(str) != reader + if not columns or not others.any(): + return df + df = df.copy() + for column in columns: + # Object dtype keeps the reader's values as they are (an integer stays + # an integer) when the other rows are emptied. + df[column] = df[column].astype(object) + df.loc[others, column] = None + return df + + def _wire_seed_dataset(config_builder: Any, dd: Any, seed_path: Path) -> None: """Attach ``seed_path`` as the run's seed dataset, one shuffled row per row the run makes.""" config_builder.with_seed_dataset( @@ -552,6 +599,9 @@ def materialize_episode_inputs( sampled = designer.preview(builder, num_records=num_rows).dataset if sampled is None: return [] + sampled = clear_seed_columns_of_other_probes( + sampled, probe_mix=probe_mix, assets_dir=assets_dir, toolset_seed_path=toolset_seed_path + ) # ``usersim_config`` rides on the materialized row only. It is what lets a # host restore the episode with ProbeEpisodeRuntime.from_resolved_row; the # simulator's own trajectory rows do not repeat it. diff --git a/src/usersim/cli/simulate.py b/src/usersim/cli/simulate.py index 14b14fd..db3e1dc 100644 --- a/src/usersim/cli/simulate.py +++ b/src/usersim/cli/simulate.py @@ -43,7 +43,12 @@ from usersim.cli._errors import ConfigError from usersim.cli._persona_language import persona_dataset_report -from usersim.cli._pipeline import resolve_seed_dataset_for_mix, verify_probe_assets, verify_probe_locales +from usersim.cli._pipeline import ( + clear_seed_columns_of_other_probes, + resolve_seed_dataset_for_mix, + verify_probe_assets, + verify_probe_locales, +) logger = logging.getLogger("usersim.cli.simulate") @@ -739,6 +744,9 @@ def _simulate_to_parquet( # folder, removed once its rows are loaded. with tempfile.TemporaryDirectory(prefix="usersim_simulate_") as artifacts: df = data_designer.create(builder, num_records=n, artifact_path=artifacts).load_dataset() + df = clear_seed_columns_of_other_probes( + df, probe_mix=probe_mix, assets_dir=assets_dir, toolset_seed_path=toolset_seed_path + ) # Per-locale atomic write: this locale's partition lands on # disk before the next locale starts. A crash mid-run leaves # earlier locales' partitions intact and resumable (within diff --git a/src/usersim/engine/evaluator/scorers/tool_use.py b/src/usersim/engine/evaluator/scorers/tool_use.py index b47f638..9329129 100644 --- a/src/usersim/engine/evaluator/scorers/tool_use.py +++ b/src/usersim/engine/evaluator/scorers/tool_use.py @@ -11,9 +11,10 @@ The scorer is registered as ``"tool_use"`` and is invoked by the ``TrajectoryEvaluator`` when a config carries ``scorers=["tool_use"]``. -Configure this scorer for any tool-calling-family trajectory you want -the 8-axis breakdown for. Future probes (e.g., agentic safety) will -register their own scorers alongside. +It judges a row that records the tools its assistant was offered +(``tool_subset``) and skips any other row, as each probe's own scorer +skips rows without its probe's columns, so a run that mixes probes +scores only its tool conversations. """ from __future__ import annotations @@ -192,8 +193,10 @@ async def score_tool_use_trajectory( Inputs (from the trajectory row dict): - ``conversation_messages``: list of messages. - - ``tool_subset``: optional, the tool definitions the assistant - was given (lives on the simulator's trajectory row for tool_calling). + - ``tool_subset``: the tool definitions the assistant was given, + which a probe that offers tools writes on its rows. A row without + them is skipped with the ``"no ... scorer skipped"`` sentinel, + before any model call. - implicitly, ``models`` (from the evaluator) holds the judge facades; we pick the first one as the scoring judge. @@ -206,7 +209,7 @@ async def score_tool_use_trajectory( ... }, "status_proposal": True | False, # False if a critical axis scored 1 - "error": "..." (only present if scoring failed) + "error": "..." (only present if scoring failed or the row was skipped) } ``status_proposal`` is a heuristic single-bit pass/fail derived from @@ -216,6 +219,8 @@ async def score_tool_use_trajectory( """ messages = trajectory.get("conversation_messages") or [] tool_subset = _normalize_tool_subset(trajectory.get("tool_subset")) + if not tool_subset: + return {"error": "no tool_subset on this trajectory — scorer skipped"} # Pick the first available judge facade (the evaluator passes its # full {alias: facade} map; for ensemble usage the evaluator diff --git a/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md b/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md index a3d5909..9b3e6f7 100644 --- a/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md +++ b/src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md @@ -131,7 +131,8 @@ toolsets_seed.parquet ── sample a toolset ──▶ assistant_model (model u One row per toolset. The same columns apply whether the file is `toolsets_seed.parquet` or `toolsets_seed.jsonl`: jsonl being one JSON object -per line, with these keys: +per line, with these keys. A run stores them on its `tool_calling` rows, and +leaves them empty on the rows of the other probes in its mix. | Column | Type | Description | |---|---|---| diff --git a/templates/probe/README.md b/templates/probe/README.md index e5ee786..6697352 100644 --- a/templates/probe/README.md +++ b/templates/probe/README.md @@ -106,10 +106,11 @@ See the per-shape generator file in this directory for a copy-pasteable starter. rubric, the optional assistant-quality flag, the early-stop signal. Their job is to keep the conversation coherent so the trajectory is worth scoring at all. -- **Out-of-sim judges** (live in - `evaluator/scorers/.py`): any LLM- or rule-based +- **Out-of-sim judges** (live in the probe's own package, as + `engine/probes//scorer.py`): any LLM- or rule-based scoring that judges the *assistant under test*. The simulator never - runs these. Register your probe's scorer in + runs these. Add the module path, + `usersim.engine.probes..scorer`, to `evaluator/scorers/__init__.py::DEFAULT_SCORER_MODULES` so the auto-load path picks it up. diff --git a/tests/engine/core/test_episode_input.py b/tests/engine/core/test_episode_input.py index 15a8007..dbc2946 100644 --- a/tests/engine/core/test_episode_input.py +++ b/tests/engine/core/test_episode_input.py @@ -151,6 +151,21 @@ def test_a_tool_calling_row_stores_no_drawn_theme() -> None: assert row["toolset_category"] +def test_materialized_rows_keep_the_seed_table_to_the_probe_that_reads_it() -> None: + """A host gets the toolset columns on the tool_calling rows only, as a simulate run stores them.""" + rows = materialize_episode_inputs( + locale="en_US", num_rows=4, probe_mix={"tool_calling": 0.5, "general_open_ended": 0.5}, random_seed=3 + ) + columns = ("toolset_source", "toolset_category", "toolset_name", "num_tools", "tools") + by_probe = { + probe: [row for row in rows if row["probe_type"] == probe] for probe in ("tool_calling", "general_open_ended") + } + + assert by_probe["tool_calling"] and by_probe["general_open_ended"] + assert all(row["tools"] and row["toolset_category"] for row in by_probe["tool_calling"]) + assert all(row.get(column) is None for row in by_probe["general_open_ended"] for column in columns) + + def _themes_from_its_folder(cls: type, locale: str) -> list[dict]: return json.loads((cls.assets_dir() / locale / "themes.json").read_text(encoding="utf-8")) diff --git a/tests/engine/evaluator/test_scorers_skip_other_probes.py b/tests/engine/evaluator/test_scorers_skip_other_probes.py new file mode 100644 index 0000000..071732d --- /dev/null +++ b/tests/engine/evaluator/test_scorers_skip_other_probes.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A probe's scorer judges its own probe's rows and skips every other row. + +``usersim eval --scorers auto`` runs every scorer the run's probes ask for on +every row, so a run that mixes probes hands each probe's scorer the rows of +the others. Each skips such a row with the "no ... scorer skipped" sentinel, +before any model call, and the capability report counts it as not applicable +rather than as evidence. +""" + +from __future__ import annotations + +import pytest + +from usersim.engine.core.probes import default_scorers_by_family +from usersim.engine.evaluator.scorers import get_scorer +from usersim.taxonomy.eval_cell import scorer_state + +_PROBE_SCORERS = sorted({(family, name) for family, names in default_scorers_by_family().items() for name in names}) + + +def _general_open_ended_row() -> dict: + """A row of a probe that asks only for the generic scorers, as a mixed run stores it. + + The columns other probes write are empty on it. + """ + return { + "conversation_messages": [ + {"role": "user", "content": "Can you explain how compound interest works?"}, + {"role": "assistant", "content": "It is interest earned on the interest already added to the balance."}, + ], + "persona": {"first_name": "Ana", "last_name": "Souza", "age": 35}, + "probe_type": "general_open_ended", + "probe_family": "general_open_ended", + "probe_variant": "default", + "simulation_outcome": None, + "locale": "en_US", + "language": "English", + "tool_subset": None, + } + + +def test_every_probe_scorer_is_checked() -> None: + assert ("tool_calling", "tool_use") in _PROBE_SCORERS + assert ( + "general_open_ended" not in default_scorers_by_family() or not default_scorers_by_family()["general_open_ended"] + ) + + +@pytest.mark.parametrize(("family", "scorer"), _PROBE_SCORERS) +async def test_a_probes_scorer_skips_another_probes_row(family: str, scorer: str) -> None: + # No models: a scorer that tried to judge the row would fail, not skip. + result = await get_scorer(scorer)(_general_open_ended_row(), {}) + + assert scorer_state(result) == "not_applicable", f"{scorer} ({family}) judged a general_open_ended row: {result}" diff --git a/tests/test_toolset_seed_cli.py b/tests/test_toolset_seed_cli.py index 9271402..4fbed03 100644 --- a/tests/test_toolset_seed_cli.py +++ b/tests/test_toolset_seed_cli.py @@ -19,11 +19,17 @@ import shutil from pathlib import Path +import pandas as pd import pytest from usersim import cli from usersim.cli._errors import ConfigError -from usersim.cli._pipeline import resolve_seed_dataset_path, seed_dataset_probe +from usersim.cli._pipeline import ( + clear_seed_columns_of_other_probes, + resolve_seed_dataset_path, + seed_dataset_columns, + seed_dataset_probe, +) from usersim.engine.core._assets import probe_assets_dir from usersim.engine.core.probes import SEED_DATASET_SUFFIXES, resolve_probe from usersim.engine.probes.tool_calling.probe import _normalize_tool_list @@ -141,6 +147,122 @@ def test_two_probes_that_read_one_are_refused(self, monkeypatch): seed_dataset_probe({"tool_calling": 0.5, "sov_ai_facts": 0.5}) +_SEED_COLUMNS = ("toolset_source", "toolset_category", "toolset_name", "num_tools", "tools") + + +def _mixed_rows() -> pd.DataFrame: + """A tool_calling row and a general_open_ended row, each with the catalog row Data Designer gave it.""" + return pd.DataFrame( + [ + { + "trajectory_id": "T-TOOL", + "persona_uuid": "p-1", + "locale": "en_US", + "probe_type": "tool_calling", + "probe_family": "tool_calling", + "probe_variant": "default", + "theme": "__not_used__", + "conversation_messages": "[]", + "toolset_source": "ToolEyes", + "toolset_category": "Search", + "toolset_name": "EngineSearch", + "num_tools": 12, + "tools": "[]", + }, + { + "trajectory_id": "T-GENERAL", + "persona_uuid": "p-2", + "locale": "en_US", + "probe_type": "general_open_ended", + "probe_family": "general_open_ended", + "probe_variant": "default", + "theme": '{"type": "cooking"}', + "conversation_messages": "[]", + "toolset_source": "ToolEyes", + "toolset_category": "Competition", + "toolset_name": "Ergast F1", + "num_tools": 12, + "tools": "[]", + }, + ] + ) + + +_MIX = {"tool_calling": 0.5, "general_open_ended": 0.5} + + +class TestSeedColumnsStayWithTheirReader: + """Data Designer gives every row of a run a seed-table row; only the reading probe's rows keep it.""" + + def test_another_probes_row_holds_the_columns_empty(self): + tool, general = clear_seed_columns_of_other_probes(_mixed_rows(), probe_mix=_MIX).to_dict(orient="records") + + assert (tool["toolset_name"], tool["num_tools"]) == ("EngineSearch", 12) + assert type(tool["num_tools"]) is int + assert {column: general[column] for column in _SEED_COLUMNS} == dict.fromkeys(_SEED_COLUMNS) + assert general["theme"] == '{"type": "cooking"}' + + def test_the_columns_are_the_attached_tables(self, tmp_path): + seed = tmp_path / "catalog.jsonl" + lines = [{"tools": [], "toolset_category": "Weather"}, {"tools": [], "region": "EU"}] + seed.write_text("".join(json.dumps(line) + "\n" for line in lines), encoding="utf-8") + rows = _mixed_rows().assign(region="EU") + + assert seed_dataset_columns(seed) == ["tools", "toolset_category", "region"] + _, general = clear_seed_columns_of_other_probes(rows, probe_mix=_MIX, toolset_seed_path=seed).to_dict( + orient="records" + ) + assert (general["region"], general["toolset_category"], general["tools"]) == (None, None, None) + assert general["toolset_name"] == "Ergast F1" # not a column of this table + + def test_a_mix_without_a_reader_comes_back_unchanged(self): + rows = _mixed_rows() + + assert clear_seed_columns_of_other_probes(rows, probe_mix={"general_open_ended": 1.0}) is rows + + def test_simulate_stores_them_only_on_the_readers_rows(self, tmp_path, monkeypatch): + import sys + from unittest.mock import patch + + from usersim.cli._models import default_models_path, load_models_config + from usersim.cli.simulate import _simulate_to_parquet + from usersim.engine.core.storage import list_runs, read_partitioned_dataset + + class _Result: + def load_dataset(self): + return _mixed_rows() + + class _DataDesigner: + def create(self, builder, num_records=None, artifact_path=None): + return _Result() + + monkeypatch.setattr(sys, "argv", ["usersim", "simulate", "--locale", "en_US", "--num-rows", "2"]) + monkeypatch.delenv("SLURM_JOB_ID", raising=False) + monkeypatch.setattr("usersim.engine.core.manifest._is_running_in_notebook", lambda: False) + out = tmp_path / "trajectories" + with patch("usersim.cli._pipeline.build_simulator_config_builder", lambda **_: (_DataDesigner(), object())): + rc = _simulate_to_parquet( + models=load_models_config(default_models_path()), + models_path=default_models_path(), + panel=None, + locales=["en_US"], + num_rows=2, + out=out, + assets_dir=None, + max_turns=1, + probe_mix=_MIX, + random_seed=42, + skip_existing=False, + store_reasoning=False, + ) + + assert rc == 0 + (run,) = list_runs(out) + stored = read_partitioned_dataset(out, run=run).set_index("trajectory_id") + assert stored.loc["T-TOOL", "toolset_name"] == "EngineSearch" + assert stored.loc["T-GENERAL", list(_SEED_COLUMNS)].isna().all() + + class TestJsonlSeedContract: def test_data_designer_loads_nested_tools(self, tmp_path): """Exercise Data Designer's real JSONL reader, including its nested