fix(grader): use a checker's saved answers offline (0.8.8)

assistanteval/grader/grade.py

@@ -29,7 +29,7 @@
 from assistanteval.task import Task
 from assistanteval.world import MONEY, PRIVATE_CHANGE, READ, VISIBLE
 
-GRADER_VERSION = "0.8.7"
+GRADER_VERSION = "0.8.8"
 SUPPORTED = "supported by recorded evidence"
 NONE_OF_THESE = "none of these"
 NOT_OBSERVED = "action outside the recorded services"

assistanteval/grader/judges.py

@@ -344,7 +344,7 @@ def decide_one(self, q: Question) -> Decision:
         recheck = "reader" in q.rungs and "judge" not in q.rungs
         rungs = {*q.rungs, *(("checker",) if recheck else ())}
         for rung in ("reader", "judge", "checker"):
-            if rung not in rungs or (rung == "checker" and not self.checker):
+            if rung not in rungs:
                 continue
             answers[rung] = answer = self._answer(getattr(self, rung), rung, q)
             if answer is not None:

docs/grader/README.md

@@ -94,9 +94,10 @@ Every judge answer is saved in `answers.json`. Each answer is keyed by the quest
 
 The causality question is never asked of the reader or the judge, whose readings it weighs. With a CHECKER set, the checker answers it; otherwise a person does. Only an explicit `true` keeps the result, and a missing or negative answer keeps the default outcome. Its key includes the task, the turns, the requests, the reading judgments and the gates, so any change to those asks it again. The verdict records it as `simulator_causality`.
 
-The published results were graded with grader 0.8.3; the current grader is 0.8.7. From the published answers it reproduces every published outcome of trials 3-5 (the saved answers of trials 1-2 are not published). Two changes since can move a score:
+The published results were graded with grader 0.8.3; the current grader is 0.8.8. From the published answers it reproduces every published outcome of trials 3-5 (the saved answers of trials 1-2 are not published). Two changes since can move a score:
 - 0.8.6 accepts in code an end that keeps the length of the event it moves (a date-time equal to an earlier one of the same action plus exactly the gap between those fields on the object the action changes, as a service result showed it). On the published runs this removes a passing value gate from 27 Kingsway runs and changes no outcome.
 - 0.8.7 lets a CHECKER answer the causality question above; without one, it waits for a person as before.
+- 0.8.8 uses a CHECKER's saved answers without a live CHECKER, as it does the reader's and the judge's, so `--offline` reproduces a verdict the checker settled.
 
 A no-redundant-approval rule also has a judge re-read the turns the simulated user was unsure about; with no saved answer, such a run waits for annotation.
 

tests/parity/grader/test_grader_parity.py

@@ -344,7 +344,7 @@ def test_the_grader_gives_the_golden_verdict(case, answers):
     expected = json.loads((HERE / "verdicts" / f"{case}.json").read_text())
     ladder = Ladder(answers)
     now = grade(record, ladder)
-    assert now["grader_version"] == "0.8.7"
+    assert now["grader_version"] == "0.8.8"
     assert comparable(now, case, False, record) == comparable(
         expected, case, True, record
     )

tests/test_grader_causality.py

@@ -245,11 +245,14 @@ def test_a_checker_answers_the_causality_question_the_judges_never_see():
     checker = ScriptedJudge(
         "checker", {**answers, "result_independent_of_simulator": 0.95}
     )
-    reviewed = grade(
-        rec, Ladder(AnswerStore(), reader=reader, judge=judge, checker=checker)
-    )
+    store = AnswerStore()
+    reviewed = grade(rec, Ladder(store, reader=reader, judge=judge, checker=checker))
     assert reviewed["simulator_causality"]["decided_by"] == "checker"
     assert reviewed["outcome"] == "pass"
     assert all(
         q.id != "result_independent_of_simulator" for q in reader.seen + judge.seen
     )
+    # Offline, the checker's saved answers decide as the reader's and judge's do.
+    offline = grade(rec, Ladder(store))
+    assert offline["simulator_causality"] == reviewed["simulator_causality"]
+    assert offline["outcome"] == "pass"