From 01ca1074e148a94c96e7eb6f93da4a8cbec2cceb Mon Sep 17 00:00:00 2001
From: Aymen Hammouda <hammouda.aymen@gmail.com>
Date: Wed, 8 Jul 2026 19:57:36 +0100
Subject: [PATCH] =?UTF-8?q?agent:=20tests=20=E2=80=94=20cover=20per-tool-m?=
 =?UTF-8?q?odel=20rollups=20and=20error-tool-call=20grounding?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Addresses both CodeRabbit nitpicks on PR #100 (additive changes to the
new test file only):

- test_score_run_produces_per_tool_model_rollups: two competitors, one
  with an explicit `id:provider/model` tool_model_key, asserting specific
  by_tool_model_key totals/scored/pending counts in the returned rollups
  and the persisted scoring-rollups.json. Adapted from CodeRabbit's
  template: the real rollup key is `by_tool_model_key`, not
  `by_tool_model`.
- test_score_run_does_not_count_error_tool_calls_as_grounding_evidence:
  a transcript whose only tool call has is_error: True does not set
  grounding_evidence_present, in the scoring record or the queue entry.

Refs #63.
---
 tests/benchmarks/test_scoring.py | 69 ++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/tests/benchmarks/test_scoring.py b/tests/benchmarks/test_scoring.py
index d74daaf..7538c9e 100644
--- a/tests/benchmarks/test_scoring.py
+++ b/tests/benchmarks/test_scoring.py
@@ -230,6 +230,29 @@ def test_score_run_detects_grounding_evidence_from_tool_calls(tmp_path: Path) ->
     assert _read_queue(run_dir)["cells"][0]["grounding_evidence_present"] is True
 
 
+def test_score_run_does_not_count_error_tool_calls_as_grounding_evidence(
+    tmp_path: Path,
+) -> None:
+    # Companion to the happy-path case above (CodeRabbit review on PR #100):
+    # a recorded tool call that errored is not grounding evidence -- only a
+    # successful call with a non-empty result payload counts
+    # (benchmarks.scoring._has_grounding_evidence).
+    scoring = _scoring_record(competitor_id="tool-a", corpus_id="SYN-EX-001")
+    transcript = _transcript_record(
+        competitor_id="tool-a",
+        corpus_id="SYN-EX-001",
+        answer="Ungrounded candidate answer.",
+        tool_calls=[{"tool": "search_docs", "arguments": {}, "result": None, "is_error": True}],
+    )
+    run_dir = _build_run_dir(tmp_path, [(scoring, transcript)])
+
+    _score(run_dir)
+
+    updated = _read_scoring(run_dir, "tool-a", "SYN-EX-001")
+    assert updated["grounding_evidence_present"] is False
+    assert _read_queue(run_dir)["cells"][0]["grounding_evidence_present"] is False
+
+
 # --- Per-category rollups + denominator invariance ----------------------------
 
 
@@ -279,6 +302,52 @@ def test_score_run_produces_per_category_rollups(tmp_path: Path) -> None:
     assert on_disk == rollups
 
 
+def test_score_run_produces_per_tool_model_rollups(tmp_path: Path) -> None:
+    # Two competitors so the by_tool_model_key breakdown is meaningfully
+    # exercised (CodeRabbit review on PR #100). tool-b carries an explicit
+    # tool_model_key (the runner's `id:provider/model` format) to verify
+    # rollups key on tool_model_key, not competitor_id.
+    cells = [
+        (
+            _scoring_record(competitor_id="tool-a", corpus_id="SYN-EX-001"),
+            _transcript_record(competitor_id="tool-a", corpus_id="SYN-EX-001", answer=""),
+        ),
+        (
+            _scoring_record(
+                competitor_id="tool-b",
+                corpus_id="SYN-EX-001",
+                tool_model_key="tool-b:openai/fake-model",
+            ),
+            _transcript_record(competitor_id="tool-b", corpus_id="SYN-EX-001", answer=""),
+        ),
+        (
+            _scoring_record(
+                competitor_id="tool-b",
+                corpus_id="SYN-EX-002",
+                tool_model_key="tool-b:openai/fake-model",
+            ),
+            _transcript_record(
+                competitor_id="tool-b", corpus_id="SYN-EX-002", answer="pending answer"
+            ),
+        ),
+    ]
+    run_dir = _build_run_dir(tmp_path, cells)
+
+    result = _score(run_dir)
+    by_tool_model = result["rollups"]["by_tool_model_key"]
+
+    assert set(by_tool_model) == {"tool-a", "tool-b:openai/fake-model"}
+    assert by_tool_model["tool-a"]["total_cells"] == 1
+    assert by_tool_model["tool-a"]["scored_cells"] == 1
+    assert by_tool_model["tool-a"]["pending_manual_scoring_cells"] == 0
+    assert by_tool_model["tool-b:openai/fake-model"]["total_cells"] == 2
+    assert by_tool_model["tool-b:openai/fake-model"]["scored_cells"] == 1
+    assert by_tool_model["tool-b:openai/fake-model"]["pending_manual_scoring_cells"] == 1
+
+    on_disk = json.loads((run_dir / "scoring-rollups.json").read_text(encoding="utf-8"))
+    assert on_disk["by_tool_model_key"] == by_tool_model
+
+
 def test_score_run_denominator_includes_every_scoring_file(tmp_path: Path) -> None:
     cells = [
         (