From 9f4f30e6150d71a5fb17f29a47821f398cfa4acb Mon Sep 17 00:00:00 2001 From: Seth Schilbe Date: Mon, 12 Oct 2026 02:00:26 +0000 Subject: [PATCH] fix(grader): use a checker's saved answers offline (0.8.8) The checker rung was skipped whenever no CHECKER was configured, before its saved answers were read, so `--offline` turned a verdict the checker settled (such as the causality question) back into one waiting for a person. Saved checker answers now decide as the reader's and judge's do. Co-Authored-By: Claude Opus 5.5 (1M context) --- assistanteval/grader/grade.py | 2 +- assistanteval/grader/judges.py | 2 +- docs/grader/README.md | 3 ++- tests/parity/grader/test_grader_parity.py | 2 +- tests/test_grader_causality.py | 9 ++++++--- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/assistanteval/grader/grade.py b/assistanteval/grader/grade.py index 80afa3f..b8de52a 100644 --- a/assistanteval/grader/grade.py +++ b/assistanteval/grader/grade.py @@ -29,7 +29,7 @@ from assistanteval.task import Task from assistanteval.world import MONEY, PRIVATE_CHANGE, READ, VISIBLE -GRADER_VERSION = "0.8.7" +GRADER_VERSION = "0.8.8" SUPPORTED = "supported by recorded evidence" NONE_OF_THESE = "none of these" NOT_OBSERVED = "action outside the recorded services" diff --git a/assistanteval/grader/judges.py b/assistanteval/grader/judges.py index 6bc43ea..a4a7cc2 100644 --- a/assistanteval/grader/judges.py +++ b/assistanteval/grader/judges.py @@ -344,7 +344,7 @@ def decide_one(self, q: Question) -> Decision: recheck = "reader" in q.rungs and "judge" not in q.rungs rungs = {*q.rungs, *(("checker",) if recheck else ())} for rung in ("reader", "judge", "checker"): - if rung not in rungs or (rung == "checker" and not self.checker): + if rung not in rungs: continue answers[rung] = answer = self._answer(getattr(self, rung), rung, q) if answer is not None: diff --git a/docs/grader/README.md b/docs/grader/README.md index 78c8829..b8b3748 100644 --- a/docs/grader/README.md +++ b/docs/grader/README.md @@ -94,9 +94,10 @@ Every judge answer is saved in `answers.json`. Each answer is keyed by the quest The causality question is never asked of the reader or the judge, whose readings it weighs. With a CHECKER set, the checker answers it; otherwise a person does. Only an explicit `true` keeps the result, and a missing or negative answer keeps the default outcome. Its key includes the task, the turns, the requests, the reading judgments and the gates, so any change to those asks it again. The verdict records it as `simulator_causality`. -The published results were graded with grader 0.8.3; the current grader is 0.8.7. From the published answers it reproduces every published outcome of trials 3-5 (the saved answers of trials 1-2 are not published). Two changes since can move a score: +The published results were graded with grader 0.8.3; the current grader is 0.8.8. From the published answers it reproduces every published outcome of trials 3-5 (the saved answers of trials 1-2 are not published). Two changes since can move a score: - 0.8.6 accepts in code an end that keeps the length of the event it moves (a date-time equal to an earlier one of the same action plus exactly the gap between those fields on the object the action changes, as a service result showed it). On the published runs this removes a passing value gate from 27 Kingsway runs and changes no outcome. - 0.8.7 lets a CHECKER answer the causality question above; without one, it waits for a person as before. +- 0.8.8 uses a CHECKER's saved answers without a live CHECKER, as it does the reader's and the judge's, so `--offline` reproduces a verdict the checker settled. A no-redundant-approval rule also has a judge re-read the turns the simulated user was unsure about; with no saved answer, such a run waits for annotation. diff --git a/tests/parity/grader/test_grader_parity.py b/tests/parity/grader/test_grader_parity.py index 01b4eac..e0db03b 100644 --- a/tests/parity/grader/test_grader_parity.py +++ b/tests/parity/grader/test_grader_parity.py @@ -344,7 +344,7 @@ def test_the_grader_gives_the_golden_verdict(case, answers): expected = json.loads((HERE / "verdicts" / f"{case}.json").read_text()) ladder = Ladder(answers) now = grade(record, ladder) - assert now["grader_version"] == "0.8.7" + assert now["grader_version"] == "0.8.8" assert comparable(now, case, False, record) == comparable( expected, case, True, record ) diff --git a/tests/test_grader_causality.py b/tests/test_grader_causality.py index 499a0ed..5139811 100644 --- a/tests/test_grader_causality.py +++ b/tests/test_grader_causality.py @@ -245,11 +245,14 @@ def test_a_checker_answers_the_causality_question_the_judges_never_see(): checker = ScriptedJudge( "checker", {**answers, "result_independent_of_simulator": 0.95} ) - reviewed = grade( - rec, Ladder(AnswerStore(), reader=reader, judge=judge, checker=checker) - ) + store = AnswerStore() + reviewed = grade(rec, Ladder(store, reader=reader, judge=judge, checker=checker)) assert reviewed["simulator_causality"]["decided_by"] == "checker" assert reviewed["outcome"] == "pass" assert all( q.id != "result_independent_of_simulator" for q in reader.seen + judge.seen ) + # Offline, the checker's saved answers decide as the reader's and judge's do. + offline = grade(rec, Ladder(store)) + assert offline["simulator_causality"] == reviewed["simulator_causality"] + assert offline["outcome"] == "pass"