From 80d5d0612f8596bc8fd372e521b5393deb4afd9e Mon Sep 17 00:00:00 2001 From: Seth Schilbe Date: Mon, 12 Oct 2026 17:36:22 +0000 Subject: [PATCH 1/2] feat(grader): a CHECKER breaks ties against the simulated user (0.8.9) When the reader or the judge is sure the simulated user misread a turn, the CHECKER answers the same question; if it is sure the user read the turn right, the user's reading stands. Otherwise the dispute goes to a person as before. Without a CHECKER nothing changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- assistanteval/endpoints.py | 4 ++- assistanteval/grader/grade.py | 10 ++++-- assistanteval/grader/judges.py | 15 ++++++++ docs/grader/README.md | 3 +- tests/parity/grader/test_grader_parity.py | 2 +- tests/test_grader.py | 43 +++++++++++++++++++++++ 7 files changed, 73 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5a64344..dbf2793 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ To run tasks in [Harbor](https://github.com/harbor-framework/harbor) against an | `READER` | Reads each assistant turn for the simulated user, and answers the grader's questions first | No; without it the judge reads too | | `WRITER` | Rewords the simulated user's fixed replies in the persona's voice | No; without it the fixed replies are sent | | `BASELINE` | The model under test in the built-in tool loop | For `--assistant model_baseline` | -| `CHECKER` | A model other than the judge that settles the grader's re-checks of the simulated user's decisions when the reader is unsure, and whether a result holds despite the user's misreadings | No; without it those go to a person | +| `CHECKER` | A model other than the judge that settles the grader's re-checks of the simulated user's decisions when the reader is unsure, breaks the tie when the reader or judge is sure the user misread a turn, and decides whether a result holds despite the user's misreadings | No; without it those go to a person | Each role reads `ASSISTANTEVAL__MODEL`, `_BASE_URL`, `_API_KEY`, and optionally `_API` (`openai`, or `jev` for TypeSafe's Jev as the reader) and `_EXTRA_BODY`. Records keep each role's model and base URL, never its key. Scores from different judge models are not directly comparable: report the judges with the scores. diff --git a/assistanteval/endpoints.py b/assistanteval/endpoints.py index a5f132c..046b355 100644 --- a/assistanteval/endpoints.py +++ b/assistanteval/endpoints.py @@ -27,7 +27,9 @@ simulated user's approval-rule decisions, which the judge made, with the reader; when the reader is unsure, the checker answers before a person does. It also answers whether a result holds despite the simulated user's - misreadings, which otherwise waits for a person. + misreadings, which otherwise waits for a person, and breaks the tie when the + reader or judge is sure the simulated user misread a turn: if the checker is + sure the user read it right, the user's reading stands. Records keep each role's model, API and base URL, never its key. """ diff --git a/assistanteval/grader/grade.py b/assistanteval/grader/grade.py index b8de52a..494a32e 100644 --- a/assistanteval/grader/grade.py +++ b/assistanteval/grader/grade.py @@ -29,7 +29,7 @@ from assistanteval.task import Task from assistanteval.world import MONEY, PRIVATE_CHANGE, READ, VISIBLE -GRADER_VERSION = "0.8.8" +GRADER_VERSION = "0.8.9" SUPPORTED = "supported by recorded evidence" NONE_OF_THESE = "none of these" NOT_OBSERVED = "action outside the recorded services" @@ -389,7 +389,11 @@ def _read_checks( def read( text: str, context: dict, confirms: bool, qid: str = f"user_read:{i}", **rungs ): - return Question(qid, "yes_no", text, context, **rungs), confirms, more + return ( + Question(qid, "yes_no", text, context, user_answer=confirms, **rungs), + confirms, + more, + ) if action == "reveal": asked = {f.id: f.reveal_when for f in user.facts} @@ -1393,6 +1397,8 @@ def _queued(d) -> dict: out = asdict(d) if out.get("checker") is None: out.pop("checker", None) + if out.get("question"): + out["question"].pop("user_answer", None) return out diff --git a/assistanteval/grader/judges.py b/assistanteval/grader/judges.py index a4a7cc2..62252ca 100644 --- a/assistanteval/grader/judges.py +++ b/assistanteval/grader/judges.py @@ -52,6 +52,9 @@ class Question: # judge already gave leaves the judge out, so it is not asked to agree # with itself; when the reader is unsure, a person decides. rungs: tuple[str, ...] = ("reader", "judge") + # For a re-read of a simulated-user decision, the answer that decision + # implies. Not part of the key. + user_answer: bool | None = None def key(self) -> str: """Same question text, criteria and context give the same key.""" @@ -333,6 +336,16 @@ def _answer(self, judge, rung: str, q: Question): self.store.put(rung, model or rung, q, answer) return answer + def _upholds_user(self, q: Question, rung: str, value, answers: dict) -> bool: + """Whether the checker breaks a tie for the simulated user: a sure + reader or judge answer against the user's reading goes to the checker, + and if it is sure the user read the turn right, the user's reading + stands, two readings to one.""" + if rung == "checker" or q.user_answer is None or value is q.user_answer: + return False + answers["checker"] = check = self._answer(self.checker, "checker", q) + return check is not None and read(q.kind, check) == (q.user_answer, True) + def decide_one(self, q: Question) -> Decision: human = self.store.get(HUMAN, q) if human is not None: @@ -349,6 +362,8 @@ def decide_one(self, q: Question) -> Decision: answers[rung] = answer = self._answer(getattr(self, rung), rung, q) if answer is not None: value, sure = read(q.kind, answer) + if sure and self._upholds_user(q, rung, value, answers): + return Decision(q.id, q.user_answer, "checker", **answers) if sure: return Decision(q.id, value, rung, **answers) note = ( diff --git a/docs/grader/README.md b/docs/grader/README.md index b8b3748..0e7a294 100644 --- a/docs/grader/README.md +++ b/docs/grader/README.md @@ -94,10 +94,11 @@ Every judge answer is saved in `answers.json`. Each answer is keyed by the quest The causality question is never asked of the reader or the judge, whose readings it weighs. With a CHECKER set, the checker answers it; otherwise a person does. Only an explicit `true` keeps the result, and a missing or negative answer keeps the default outcome. Its key includes the task, the turns, the requests, the reading judgments and the gates, so any change to those asks it again. The verdict records it as `simulator_causality`. -The published results were graded with grader 0.8.3; the current grader is 0.8.8. From the published answers it reproduces every published outcome of trials 3-5 (the saved answers of trials 1-2 are not published). Two changes since can move a score: +The published results were graded with grader 0.8.3; the current grader is 0.8.9. From the published answers it reproduces every published outcome of trials 3-5 (the saved answers of trials 1-2 are not published). Two changes since can move a score: - 0.8.6 accepts in code an end that keeps the length of the event it moves (a date-time equal to an earlier one of the same action plus exactly the gap between those fields on the object the action changes, as a service result showed it). On the published runs this removes a passing value gate from 27 Kingsway runs and changes no outcome. - 0.8.7 lets a CHECKER answer the causality question above; without one, it waits for a person as before. - 0.8.8 uses a CHECKER's saved answers without a live CHECKER, as it does the reader's and the judge's, so `--offline` reproduces a verdict the checker settled. +- 0.8.9 has a CHECKER break the tie when the reader or the judge is sure the simulated user misread a turn: if the checker is sure the user read it right, the user's reading stands; otherwise the dispute goes to a person as before. Without a CHECKER nothing changes. A no-redundant-approval rule also has a judge re-read the turns the simulated user was unsure about; with no saved answer, such a run waits for annotation. diff --git a/tests/parity/grader/test_grader_parity.py b/tests/parity/grader/test_grader_parity.py index e0db03b..4e81b93 100644 --- a/tests/parity/grader/test_grader_parity.py +++ b/tests/parity/grader/test_grader_parity.py @@ -344,7 +344,7 @@ def test_the_grader_gives_the_golden_verdict(case, answers): expected = json.loads((HERE / "verdicts" / f"{case}.json").read_text()) ladder = Ladder(answers) now = grade(record, ladder) - assert now["grader_version"] == "0.8.8" + assert now["grader_version"] == "0.8.9" assert comparable(now, case, False, record) == comparable( expected, case, True, record ) diff --git a/tests/test_grader.py b/tests/test_grader.py index 264305e..81619d1 100644 --- a/tests/test_grader.py +++ b/tests/test_grader.py @@ -1,5 +1,6 @@ import copy import json +from dataclasses import replace from datetime import datetime from pathlib import Path @@ -2145,6 +2146,48 @@ def __call__(self, q): return self.p +@pytest.mark.parametrize( + "reader, judge, checker, decided", + [ + # The reader is sure the user misread; the checker is sure it did not. + (0.1, None, 0.9, (True, "checker")), + # The judge settles it against the user; the checker breaks the tie. + (0.5, 0.05, 0.95, (True, "checker")), + # The checker agrees with the reader, or is unsure: the dispute stands. + (0.1, None, 0.1, (False, "reader")), + (0.1, None, 0.5, (False, "reader")), + # No checker: as before. + (0.1, None, None, (False, "reader")), + ], +) +def test_a_checker_breaks_a_tie_against_the_simulated_users_reading( + reader, judge, checker, decided +): + """A sure reader or judge answer against the simulated user's reading goes + to the CHECKER: if it is sure the user read the turn right, the user's + reading stands.""" + q = Question("user_read:0", "yes_no", "Does it ask?", {}, user_answer=True) + ladder = Ladder( + AnswerStore(), + reader=Fixed(reader, "reader"), + judge=Fixed(judge, "judge") if judge is not None else None, + checker=Fixed(checker, "checker") if checker is not None else None, + ) + d = ladder.decide_one(q) + assert (d.value, d.decided_by) == decided + # The checker is never asked when the reading agrees with the user's. + agreeing = Ladder( + AnswerStore(), reader=Fixed(0.9, "reader"), checker=Fixed(0.1, "checker") + ) + assert agreeing.decide_one(q).checker is None + + +def test_a_tie_break_keeps_question_keys(): + """The simulated user's answer is not part of a question's key, so saved answers still match.""" + plain = Question("user_read:0", "yes_no", "Does it ask?", {"t": 1}) + assert replace(plain, user_answer=False).key() == plain.key() + + def test_a_checker_answers_a_recheck_the_reader_is_unsure_of(): """A re-check that leaves the judge out goes to the CHECKER, if set, before a person; other questions never reach it.""" From 677bb94223d1348c676b3aece6f7ee4bfab1c498 Mon Sep 17 00:00:00 2001 From: Seth Schilbe Date: Mon, 12 Oct 2026 22:50:25 +0000 Subject: [PATCH 2/2] fix(grader): keep user_answer out of recorded questions The causality question's context embeds each disputed question, so the new field changed its key and saved causality answers stopped matching. Questions are now recorded without it, in that context and in the annotation queue. Co-Authored-By: Claude Opus 5.5 (1M context) --- assistanteval/grader/grade.py | 6 ++---- assistanteval/grader/judges.py | 8 +++++++- tests/test_grader_causality.py | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/assistanteval/grader/grade.py b/assistanteval/grader/grade.py index 494a32e..fc029c3 100644 --- a/assistanteval/grader/grade.py +++ b/assistanteval/grader/grade.py @@ -1336,7 +1336,7 @@ def _causality_question( "simulated_user_decisions": _decisions(record), "reading_judgments": [ { - "question": asdict(ask.asked[d.question_id]), + "question": ask.asked[d.question_id].recorded(), "answer": d.value, "decided_by": d.decided_by, } @@ -1394,11 +1394,9 @@ def _measures(record: RunRecord, private: list[tuple[Request, Decision]]) -> dic def _queued(d) -> dict: """A decision for the annotation queue; a checker's answer only if one was asked.""" - out = asdict(d) + out = asdict(d) | {"question": d.question and d.question.recorded()} if out.get("checker") is None: out.pop("checker", None) - if out.get("question"): - out["question"].pop("user_answer", None) return out diff --git a/assistanteval/grader/judges.py b/assistanteval/grader/judges.py index 62252ca..48d523b 100644 --- a/assistanteval/grader/judges.py +++ b/assistanteval/grader/judges.py @@ -16,7 +16,7 @@ import urllib.error import urllib.request from concurrent.futures import Future, ThreadPoolExecutor -from dataclasses import dataclass, field, replace +from dataclasses import asdict, dataclass, field, replace from pathlib import Path from assistanteval.endpoints import Endpoint, endpoint, required @@ -56,6 +56,12 @@ class Question: # implies. Not part of the key. user_answer: bool | None = None + def recorded(self) -> dict: + """The question as asked and recorded: `user_answer` only steers the ladder.""" + out = asdict(self) + del out["user_answer"] + return out + def key(self) -> str: """Same question text, criteria and context give the same key.""" return sha256(canonical([self.kind, self.text, self.criteria, self.context]))[ diff --git a/tests/test_grader_causality.py b/tests/test_grader_causality.py index 5139811..28f982f 100644 --- a/tests/test_grader_causality.py +++ b/tests/test_grader_causality.py @@ -235,6 +235,24 @@ def test_uncertain_live_readings_are_rechecked_without_assuming_their_top_choice assert gate == result +def test_a_causality_answer_saved_before_the_tie_break_still_replays(): + """The key of this case's causality question under grader 0.8.8: a saved + answer to it must still decide the run, so nothing the ladder uses + internally may enter the question's context.""" + rec, answers, _ = case("pass") + answers["user_read"] = 0.5 + store = AnswerStore() + store.data["answers"][f"{HUMAN}:a26041974bdea0dcacc266e6c1bb6173"] = { + "answer": True, + "judge": HUMAN, + "model": HUMAN, + "question_id": "result_independent_of_simulator", + } + reviewed = run(rec, answers, store=store) + assert reviewed["simulator_causality"]["decided_by"] == HUMAN + assert reviewed["outcome"] == "pass" + + def test_a_checker_answers_the_causality_question_the_judges_never_see(): """With a CHECKER, its yes keeps the result a disputed reading would void; the reader and the judge are still never asked."""