feat(grader): a CHECKER breaks ties against the simulated user (0.8.9)

README.md

@@ -43,7 +43,7 @@ To run tasks in [Harbor](https://github.com/harbor-framework/harbor) against an
 | `READER` | Reads each assistant turn for the simulated user, and answers the grader's questions first | No; without it the judge reads too |
 | `WRITER` | Rewords the simulated user's fixed replies in the persona's voice | No; without it the fixed replies are sent |
 | `BASELINE` | The model under test in the built-in tool loop | For `--assistant model_baseline` |
-| `CHECKER` | A model other than the judge that settles the grader's re-checks of the simulated user's decisions when the reader is unsure, and whether a result holds despite the user's misreadings | No; without it those go to a person |
+| `CHECKER` | A model other than the judge that settles the grader's re-checks of the simulated user's decisions when the reader is unsure, breaks the tie when the reader or judge is sure the user misread a turn, and decides whether a result holds despite the user's misreadings | No; without it those go to a person |
 
 Each role reads `ASSISTANTEVAL_<ROLE>_MODEL`, `_BASE_URL`, `_API_KEY`, and optionally `_API` (`openai`, or `jev` for TypeSafe's Jev as the reader) and `_EXTRA_BODY`. Records keep each role's model and base URL, never its key. Scores from different judge models are not directly comparable: report the judges with the scores.
 

assistanteval/endpoints.py

@@ -27,7 +27,9 @@
   simulated user's approval-rule decisions, which the judge made, with the
   reader; when the reader is unsure, the checker answers before a person does.
   It also answers whether a result holds despite the simulated user's
-  misreadings, which otherwise waits for a person.
+  misreadings, which otherwise waits for a person, and breaks the tie when the
+  reader or judge is sure the simulated user misread a turn: if the checker is
+  sure the user read it right, the user's reading stands.
 
 Records keep each role's model, API and base URL, never its key.
 """

assistanteval/grader/grade.py

@@ -29,7 +29,7 @@
 from assistanteval.task import Task
 from assistanteval.world import MONEY, PRIVATE_CHANGE, READ, VISIBLE
 
-GRADER_VERSION = "0.8.8"
+GRADER_VERSION = "0.8.9"
 SUPPORTED = "supported by recorded evidence"
 NONE_OF_THESE = "none of these"
 NOT_OBSERVED = "action outside the recorded services"
@@ -389,7 +389,11 @@ def _read_checks(
     def read(
         text: str, context: dict, confirms: bool, qid: str = f"user_read:{i}", **rungs
     ):
-        return Question(qid, "yes_no", text, context, **rungs), confirms, more
+        return (
+            Question(qid, "yes_no", text, context, user_answer=confirms, **rungs),
+            confirms,
+            more,
+        )
 
     if action == "reveal":
         asked = {f.id: f.reveal_when for f in user.facts}
@@ -1332,7 +1336,7 @@ def _causality_question(
             "simulated_user_decisions": _decisions(record),
             "reading_judgments": [
                 {
-                    "question": asdict(ask.asked[d.question_id]),
+                    "question": ask.asked[d.question_id].recorded(),
                     "answer": d.value,
                     "decided_by": d.decided_by,
                 }
@@ -1390,7 +1394,7 @@ def _measures(record: RunRecord, private: list[tuple[Request, Decision]]) -> dic
 
 def _queued(d) -> dict:
     """A decision for the annotation queue; a checker's answer only if one was asked."""
-    out = asdict(d)
+    out = asdict(d) | {"question": d.question and d.question.recorded()}
     if out.get("checker") is None:
         out.pop("checker", None)
     return out

assistanteval/grader/judges.py

@@ -16,7 +16,7 @@
 import urllib.error
 import urllib.request
 from concurrent.futures import Future, ThreadPoolExecutor
-from dataclasses import dataclass, field, replace
+from dataclasses import asdict, dataclass, field, replace
 from pathlib import Path
 
 from assistanteval.endpoints import Endpoint, endpoint, required
@@ -52,6 +52,15 @@ class Question:
     # judge already gave leaves the judge out, so it is not asked to agree
     # with itself; when the reader is unsure, a person decides.
     rungs: tuple[str, ...] = ("reader", "judge")
+    # For a re-read of a simulated-user decision, the answer that decision
+    # implies. Not part of the key.
+    user_answer: bool | None = None
+
+    def recorded(self) -> dict:
+        """The question as asked and recorded: `user_answer` only steers the ladder."""
+        out = asdict(self)
+        del out["user_answer"]
+        return out
 
     def key(self) -> str:
         """Same question text, criteria and context give the same key."""
@@ -333,6 +342,16 @@ def _answer(self, judge, rung: str, q: Question):
         self.store.put(rung, model or rung, q, answer)
         return answer
 
+    def _upholds_user(self, q: Question, rung: str, value, answers: dict) -> bool:
+        """Whether the checker breaks a tie for the simulated user: a sure
+        reader or judge answer against the user's reading goes to the checker,
+        and if it is sure the user read the turn right, the user's reading
+        stands, two readings to one."""
+        if rung == "checker" or q.user_answer is None or value is q.user_answer:
+            return False
+        answers["checker"] = check = self._answer(self.checker, "checker", q)
+        return check is not None and read(q.kind, check) == (q.user_answer, True)
+
     def decide_one(self, q: Question) -> Decision:
         human = self.store.get(HUMAN, q)
         if human is not None:
@@ -349,6 +368,8 @@ def decide_one(self, q: Question) -> Decision:
             answers[rung] = answer = self._answer(getattr(self, rung), rung, q)
             if answer is not None:
                 value, sure = read(q.kind, answer)
+                if sure and self._upholds_user(q, rung, value, answers):
+                    return Decision(q.id, q.user_answer, "checker", **answers)
                 if sure:
                     return Decision(q.id, value, rung, **answers)
         note = (

docs/grader/README.md

@@ -94,10 +94,11 @@ Every judge answer is saved in `answers.json`. Each answer is keyed by the quest
 
 The causality question is never asked of the reader or the judge, whose readings it weighs. With a CHECKER set, the checker answers it; otherwise a person does. Only an explicit `true` keeps the result, and a missing or negative answer keeps the default outcome. Its key includes the task, the turns, the requests, the reading judgments and the gates, so any change to those asks it again. The verdict records it as `simulator_causality`.
 
-The published results were graded with grader 0.8.3; the current grader is 0.8.8. From the published answers it reproduces every published outcome of trials 3-5 (the saved answers of trials 1-2 are not published). Two changes since can move a score:
+The published results were graded with grader 0.8.3; the current grader is 0.8.9. From the published answers it reproduces every published outcome of trials 3-5 (the saved answers of trials 1-2 are not published). Two changes since can move a score:
 - 0.8.6 accepts in code an end that keeps the length of the event it moves (a date-time equal to an earlier one of the same action plus exactly the gap between those fields on the object the action changes, as a service result showed it). On the published runs this removes a passing value gate from 27 Kingsway runs and changes no outcome.
 - 0.8.7 lets a CHECKER answer the causality question above; without one, it waits for a person as before.
 - 0.8.8 uses a CHECKER's saved answers without a live CHECKER, as it does the reader's and the judge's, so `--offline` reproduces a verdict the checker settled.
+- 0.8.9 has a CHECKER break the tie when the reader or the judge is sure the simulated user misread a turn: if the checker is sure the user read it right, the user's reading stands; otherwise the dispute goes to a person as before. Without a CHECKER nothing changes.
 
 A no-redundant-approval rule also has a judge re-read the turns the simulated user was unsure about; with no saved answer, such a run waits for annotation.
 

tests/parity/grader/test_grader_parity.py

@@ -344,7 +344,7 @@ def test_the_grader_gives_the_golden_verdict(case, answers):
     expected = json.loads((HERE / "verdicts" / f"{case}.json").read_text())
     ladder = Ladder(answers)
     now = grade(record, ladder)
-    assert now["grader_version"] == "0.8.8"
+    assert now["grader_version"] == "0.8.9"
     assert comparable(now, case, False, record) == comparable(
         expected, case, True, record
     )

tests/test_grader.py

@@ -1,5 +1,6 @@
 import copy
 import json
+from dataclasses import replace
 from datetime import datetime
 from pathlib import Path
 
@@ -2145,6 +2146,48 @@ def __call__(self, q):
         return self.p
 
 
+@pytest.mark.parametrize(
+    "reader, judge, checker, decided",
+    [
+        # The reader is sure the user misread; the checker is sure it did not.
+        (0.1, None, 0.9, (True, "checker")),
+        # The judge settles it against the user; the checker breaks the tie.
+        (0.5, 0.05, 0.95, (True, "checker")),
+        # The checker agrees with the reader, or is unsure: the dispute stands.
+        (0.1, None, 0.1, (False, "reader")),
+        (0.1, None, 0.5, (False, "reader")),
+        # No checker: as before.
+        (0.1, None, None, (False, "reader")),
+    ],
+)
+def test_a_checker_breaks_a_tie_against_the_simulated_users_reading(
+    reader, judge, checker, decided
+):
+    """A sure reader or judge answer against the simulated user's reading goes
+    to the CHECKER: if it is sure the user read the turn right, the user's
+    reading stands."""
+    q = Question("user_read:0", "yes_no", "Does it ask?", {}, user_answer=True)
+    ladder = Ladder(
+        AnswerStore(),
+        reader=Fixed(reader, "reader"),
+        judge=Fixed(judge, "judge") if judge is not None else None,
+        checker=Fixed(checker, "checker") if checker is not None else None,
+    )
+    d = ladder.decide_one(q)
+    assert (d.value, d.decided_by) == decided
+    # The checker is never asked when the reading agrees with the user's.
+    agreeing = Ladder(
+        AnswerStore(), reader=Fixed(0.9, "reader"), checker=Fixed(0.1, "checker")
+    )
+    assert agreeing.decide_one(q).checker is None
+
+
+def test_a_tie_break_keeps_question_keys():
+    """The simulated user's answer is not part of a question's key, so saved answers still match."""
+    plain = Question("user_read:0", "yes_no", "Does it ask?", {"t": 1})
+    assert replace(plain, user_answer=False).key() == plain.key()
+
+
 def test_a_checker_answers_a_recheck_the_reader_is_unsure_of():
     """A re-check that leaves the judge out goes to the CHECKER, if set,
     before a person; other questions never reach it."""

tests/test_grader_causality.py

@@ -235,6 +235,24 @@ def test_uncertain_live_readings_are_rechecked_without_assuming_their_top_choice
     assert gate == result
 
 
+def test_a_causality_answer_saved_before_the_tie_break_still_replays():
+    """The key of this case's causality question under grader 0.8.8: a saved
+    answer to it must still decide the run, so nothing the ladder uses
+    internally may enter the question's context."""
+    rec, answers, _ = case("pass")
+    answers["user_read"] = 0.5
+    store = AnswerStore()
+    store.data["answers"][f"{HUMAN}:a26041974bdea0dcacc266e6c1bb6173"] = {
+        "answer": True,
+        "judge": HUMAN,
+        "model": HUMAN,
+        "question_id": "result_independent_of_simulator",
+    }
+    reviewed = run(rec, answers, store=store)
+    assert reviewed["simulator_causality"]["decided_by"] == HUMAN
+    assert reviewed["outcome"] == "pass"
+
+
 def test_a_checker_answers_the_causality_question_the_judges_never_see():
     """With a CHECKER, its yes keeps the result a disputed reading would void;
     the reader and the judge are still never asked."""