fix(engine): keep each probe's scorer and seed data to its own rows

architecture/evaluator.md

@@ -44,7 +44,9 @@ under; `auto` combines with scorer names. Over a run holding every built-in
 probe it runs every scorer the capability report reads. A capability whose
 scorer was not dispatched renders as *untested* rather than as a failure, and
 `capabilities_missing_scorer()` exists so a run can warn about that instead
-of silently producing an empty column.
+of silently producing an empty column. The evaluator calls every scorer it runs
+on every row, so a probe's scorer skips the rows of other probes, and the
+capability report counts a skipped row as not applicable.
 
 ## The evaluation cell
 

architecture/probes.md

@@ -135,7 +135,7 @@ than into a branch on its label.
 | `authoring_guide` | `None` | the pre-flight, when a run asks for a locale the probe has no bank for |
 | `asset_problems` | no problems | `usersim smoke` and `assert_probe_conforms` |
 | `themes` | `None`: no theme | run setup, which draws one per row into the `theme` column |
-| `seed_dataset` | `None`: no seed table | run setup, which attaches `<seed_dataset>.parquet` or `.jsonl` from the probe's asset folder and gives each row the columns of one of its rows; a mix with two such probes is refused; smoke and `assert_probe_conforms` check the table is there |
+| `seed_dataset` | `None`: no seed table | run setup, which attaches `<seed_dataset>.parquet` or `.jsonl` from the probe's asset folder and gives each of the probe's rows the columns of one of its rows, and the mix's other probes' rows those columns empty; a mix with two such probes is refused; smoke and `assert_probe_conforms` check the table is there |
 | `model_aliases` | `()` | run setup, which adds each alias the models file defines to every row's models |
 | `output_columns` | `()` | the run's `side_effect_columns`; a column left out is dropped |
 | `supports_assistant_resampling` | `True`; `False` with the tool mixins | judge-gated assistant resampling (`max_assistant_attempts > 1`) |

docs/engine/AUTHORING_A_PROBE.md

@@ -1282,7 +1282,11 @@ rows, beside the rule-based `language_compliance`, `refusal_basics` and
 reports a name in `default_scorers` that no registered scorer holds. The
 evaluator calls each scorer it runs on every row, so return early for rows from
 other probes, for example when your probe's join key is missing, as the
-`safety_agentic` scorer does with `action_request_id`.
+`safety_agentic` scorer does with `action_request_id`. Return an empty mapping,
+or an `error` that starts with `no ` and contains `scorer skipped`: the
+capability report counts the row as not applicable rather than as evidence.
+[`tests/engine/evaluator/test_scorers_skip_other_probes.py`](../../tests/engine/evaluator/test_scorers_skip_other_probes.py)
+checks that every built-in probe's scorer skips another probe's row.
 
 A scorer in your probe's package may write its label;
 [`tests/test_probe_names_in_shared_code.py`](../../tests/test_probe_names_in_shared_code.py)

docs/engine/README.md

@@ -431,7 +431,8 @@ hook), see the source-of-truth docstrings in
 **Assistant evaluation is configured separately** via
 `TrajectoryEvaluatorConfig` (see Trajectory Evaluator section). Add
 new axes to `evaluator/axes.py::PROBE_SCORES` if needed; add a
-scorer under `evaluator/scorers/your_probe.py` and include it in
+scorer as `scorer.py` in the probe's package
+(`engine/probes/your_probe/scorer.py`) and include its module path in
 `DEFAULT_SCORER_MODULES` to register it on import; name it in `--scorers` to
 run it, or list it in the probe's `default_scorers` so `--scorers auto` runs
 it.

notebooks/01_simulate.ipynb

@@ -94,7 +94,12 @@
     ")\n",
     "from usersim.cli._env import describe_loaded_env, load_local_env\n",
     "from usersim.cli._ngc import ensure_ngc_cli, ensure_ngc_org, ngc_available\n",
-    "from usersim.cli._pipeline import build_simulator_config_builder, known_probes, sample_trajectories\n",
+    "from usersim.cli._pipeline import (\n",
+    "    build_simulator_config_builder,\n",
+    "    clear_seed_columns_of_other_probes,\n",
+    "    known_probes,\n",
+    "    sample_trajectories,\n",
+    ")\n",
     "\n",
     "# --- Engine (asset + storage + preview helpers) ---\n",
     "from usersim.engine.core._assets import probe_assets_dir\n",
@@ -704,6 +709,11 @@
     "    with tempfile.TemporaryDirectory(prefix=\"usersim_simulate_\") as _artifacts:\n",
     "        result = data_designer.create(builder, num_records=NUM_ROWS_PER_LOCALE, artifact_path=_artifacts)\n",
     "        df = result.load_dataset()\n",
+    "    # Data Designer gives every row the seed table's columns; keep them to the\n",
+    "    # probe that reads the table.\n",
+    "    df = clear_seed_columns_of_other_probes(\n",
+    "        df, probe_mix=PROBE_MIX, assets_dir=ASSETS_DIR, toolset_seed_path=_seed_path\n",
+    "    )\n",
     "    # Atomic per-locale write: this locale's partition lands on disk\n",
     "    # before the next locale starts. A crash mid-run leaves earlier\n",
     "    # locales' partitions intact and resumable. It holds the run lock, skips\n",

src/usersim/cli/_pipeline.py

@@ -321,6 +321,53 @@ def resolve_seed_dataset_for_mix(
     return resolved
 
 
+def seed_dataset_columns(seed_path: Path) -> list[str]:
+    """The columns of the seed table at ``seed_path``: a parquet file's schema, or every key of a JSONL file."""
+    if seed_path.suffix == ".parquet":
+        import pyarrow.parquet as pq
+
+        return list(pq.read_schema(seed_path).names)
+    columns: dict[str, None] = {}
+    with seed_path.open(encoding="utf-8") as lines:
+        for line in lines:
+            if line.strip():
+                columns.update(dict.fromkeys(json.loads(line)))
+    return list(columns)
+
+
+def clear_seed_columns_of_other_probes(
+    df: Any,
+    *,
+    probe_mix: Mapping[str, float],
+    assets_dir: Path | str | None = None,
+    toolset_seed_path: Path | str | None = None,
+) -> Any:
+    """``df`` with the seed table's columns emptied on every row of a probe that does not read the table.
+
+    Data Designer gives each row of a run the columns of one seed-table row,
+    whatever the row's probe, so without this a row of another probe in the
+    mix would record the seed row it drew: a toolset its conversation never
+    offered, say. The table is the one the run attaches, resolved from the
+    same arguments, and ``df`` comes back unchanged when no probe in the mix
+    reads one.
+    """
+    seed_path = resolve_seed_dataset_for_mix(probe_mix, assets_dir, toolset_seed_path)
+    reader = seed_dataset_probe(probe_mix)
+    if seed_path is None or reader is None or "probe_type" not in df.columns:
+        return df
+    columns = [column for column in seed_dataset_columns(seed_path) if column in df.columns]
+    others = df["probe_type"].astype(str) != reader
+    if not columns or not others.any():
+        return df
+    df = df.copy()
+    for column in columns:
+        # Object dtype keeps the reader's values as they are (an integer stays
+        # an integer) when the other rows are emptied.
+        df[column] = df[column].astype(object)
+        df.loc[others, column] = None
+    return df
+
+
 def _wire_seed_dataset(config_builder: Any, dd: Any, seed_path: Path) -> None:
     """Attach ``seed_path`` as the run's seed dataset, one shuffled row per row the run makes."""
     config_builder.with_seed_dataset(
@@ -552,6 +599,9 @@ def materialize_episode_inputs(
     sampled = designer.preview(builder, num_records=num_rows).dataset
     if sampled is None:
         return []
+    sampled = clear_seed_columns_of_other_probes(
+        sampled, probe_mix=probe_mix, assets_dir=assets_dir, toolset_seed_path=toolset_seed_path
+    )
     # ``usersim_config`` rides on the materialized row only. It is what lets a
     # host restore the episode with ProbeEpisodeRuntime.from_resolved_row; the
     # simulator's own trajectory rows do not repeat it.

src/usersim/cli/simulate.py

@@ -43,7 +43,12 @@
 
 from usersim.cli._errors import ConfigError
 from usersim.cli._persona_language import persona_dataset_report
-from usersim.cli._pipeline import resolve_seed_dataset_for_mix, verify_probe_assets, verify_probe_locales
+from usersim.cli._pipeline import (
+    clear_seed_columns_of_other_probes,
+    resolve_seed_dataset_for_mix,
+    verify_probe_assets,
+    verify_probe_locales,
+)
 
 logger = logging.getLogger("usersim.cli.simulate")
 
@@ -739,6 +744,9 @@ def _simulate_to_parquet(
         # folder, removed once its rows are loaded.
         with tempfile.TemporaryDirectory(prefix="usersim_simulate_") as artifacts:
             df = data_designer.create(builder, num_records=n, artifact_path=artifacts).load_dataset()
+        df = clear_seed_columns_of_other_probes(
+            df, probe_mix=probe_mix, assets_dir=assets_dir, toolset_seed_path=toolset_seed_path
+        )
         # Per-locale atomic write: this locale's partition lands on
         # disk before the next locale starts. A crash mid-run leaves
         # earlier locales' partitions intact and resumable (within

src/usersim/engine/evaluator/scorers/tool_use.py

@@ -11,9 +11,10 @@
 
 The scorer is registered as ``"tool_use"`` and is invoked by the
 ``TrajectoryEvaluator`` when a config carries ``scorers=["tool_use"]``.
-Configure this scorer for any tool-calling-family trajectory you want
-the 8-axis breakdown for. Future probes (e.g., agentic safety) will
-register their own scorers alongside.
+It judges a row that records the tools its assistant was offered
+(``tool_subset``) and skips any other row, as each probe's own scorer
+skips rows without its probe's columns, so a run that mixes probes
+scores only its tool conversations.
 """
 
 from __future__ import annotations
@@ -192,8 +193,10 @@ async def score_tool_use_trajectory(
 
     Inputs (from the trajectory row dict):
       - ``conversation_messages``: list of messages.
-      - ``tool_subset``: optional, the tool definitions the assistant
-        was given (lives on the simulator's trajectory row for tool_calling).
+      - ``tool_subset``: the tool definitions the assistant was given,
+        which a probe that offers tools writes on its rows. A row without
+        them is skipped with the ``"no ... scorer skipped"`` sentinel,
+        before any model call.
       - implicitly, ``models`` (from the evaluator) holds the judge
         facades; we pick the first one as the scoring judge.
 
@@ -206,7 +209,7 @@ async def score_tool_use_trajectory(
             ...
           },
           "status_proposal": True | False,   # False if a critical axis scored 1
-          "error": "..."  (only present if scoring failed)
+          "error": "..."  (only present if scoring failed or the row was skipped)
         }
 
     ``status_proposal`` is a heuristic single-bit pass/fail derived from
@@ -216,6 +219,8 @@ async def score_tool_use_trajectory(
     """
     messages = trajectory.get("conversation_messages") or []
     tool_subset = _normalize_tool_subset(trajectory.get("tool_subset"))
+    if not tool_subset:
+        return {"error": "no tool_subset on this trajectory — scorer skipped"}
 
     # Pick the first available judge facade (the evaluator passes its
     # full {alias: facade} map; for ensemble usage the evaluator

src/usersim/engine/probes/tool_calling/assets/tool_calling/README.md

@@ -131,7 +131,8 @@ toolsets_seed.parquet ── sample a toolset ──▶ assistant_model (model u
 
 One row per toolset. The same columns apply whether the file is
 `toolsets_seed.parquet` or `toolsets_seed.jsonl`: jsonl being one JSON object
-per line, with these keys:
+per line, with these keys. A run stores them on its `tool_calling` rows, and
+leaves them empty on the rows of the other probes in its mix.
 
 | Column | Type | Description |
 |---|---|---|

templates/probe/README.md

@@ -106,10 +106,11 @@ See the per-shape generator file in this directory for a copy-pasteable starter.
   rubric, the optional assistant-quality flag, the early-stop signal.
   Their job is to keep the conversation coherent so the trajectory is
   worth scoring at all.
-- **Out-of-sim judges** (live in
-  `evaluator/scorers/<your_probe>.py`): any LLM- or rule-based
+- **Out-of-sim judges** (live in the probe's own package, as
+  `engine/probes/<your_probe>/scorer.py`): any LLM- or rule-based
   scoring that judges the *assistant under test*. The simulator never
-  runs these. Register your probe's scorer in
+  runs these. Add the module path,
+  `usersim.engine.probes.<your_probe>.scorer`, to
   `evaluator/scorers/__init__.py::DEFAULT_SCORER_MODULES` so the
   auto-load path picks it up.
 

tests/engine/core/test_episode_input.py

@@ -151,6 +151,21 @@ def test_a_tool_calling_row_stores_no_drawn_theme() -> None:
     assert row["toolset_category"]
 
 
+def test_materialized_rows_keep_the_seed_table_to_the_probe_that_reads_it() -> None:
+    """A host gets the toolset columns on the tool_calling rows only, as a simulate run stores them."""
+    rows = materialize_episode_inputs(
+        locale="en_US", num_rows=4, probe_mix={"tool_calling": 0.5, "general_open_ended": 0.5}, random_seed=3
+    )
+    columns = ("toolset_source", "toolset_category", "toolset_name", "num_tools", "tools")
+    by_probe = {
+        probe: [row for row in rows if row["probe_type"] == probe] for probe in ("tool_calling", "general_open_ended")
+    }
+
+    assert by_probe["tool_calling"] and by_probe["general_open_ended"]
+    assert all(row["tools"] and row["toolset_category"] for row in by_probe["tool_calling"])
+    assert all(row.get(column) is None for row in by_probe["general_open_ended"] for column in columns)
+
+
 def _themes_from_its_folder(cls: type, locale: str) -> list[dict]:
     return json.loads((cls.assets_dir() / locale / "themes.json").read_text(encoding="utf-8"))
 

tests/engine/evaluator/test_scorers_skip_other_probes.py

@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""A probe's scorer judges its own probe's rows and skips every other row.
+
+``usersim eval --scorers auto`` runs every scorer the run's probes ask for on
+every row, so a run that mixes probes hands each probe's scorer the rows of
+the others. Each skips such a row with the "no ... scorer skipped" sentinel,
+before any model call, and the capability report counts it as not applicable
+rather than as evidence.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from usersim.engine.core.probes import default_scorers_by_family
+from usersim.engine.evaluator.scorers import get_scorer
+from usersim.taxonomy.eval_cell import scorer_state
+
+_PROBE_SCORERS = sorted({(family, name) for family, names in default_scorers_by_family().items() for name in names})
+
+
+def _general_open_ended_row() -> dict:
+    """A row of a probe that asks only for the generic scorers, as a mixed run stores it.
+
+    The columns other probes write are empty on it.
+    """
+    return {
+        "conversation_messages": [
+            {"role": "user", "content": "Can you explain how compound interest works?"},
+            {"role": "assistant", "content": "It is interest earned on the interest already added to the balance."},
+        ],
+        "persona": {"first_name": "Ana", "last_name": "Souza", "age": 35},
+        "probe_type": "general_open_ended",
+        "probe_family": "general_open_ended",
+        "probe_variant": "default",
+        "simulation_outcome": None,
+        "locale": "en_US",
+        "language": "English",
+        "tool_subset": None,
+    }
+
+
+def test_every_probe_scorer_is_checked() -> None:
+    assert ("tool_calling", "tool_use") in _PROBE_SCORERS
+    assert (
+        "general_open_ended" not in default_scorers_by_family() or not default_scorers_by_family()["general_open_ended"]
+    )
+
+
+@pytest.mark.parametrize(("family", "scorer"), _PROBE_SCORERS)
+async def test_a_probes_scorer_skips_another_probes_row(family: str, scorer: str) -> None:
+    # No models: a scorer that tried to judge the row would fail, not skip.
+    result = await get_scorer(scorer)(_general_open_ended_row(), {})
+
+    assert scorer_state(result) == "not_applicable", f"{scorer} ({family}) judged a general_open_ended row: {result}"