From dff6d90e8e0140fbdfda9c780da6b8a51e3e6cfd Mon Sep 17 00:00:00 2001 From: Seth Schilbe Date: Mon, 12 Oct 2026 23:29:56 +0000 Subject: [PATCH 1/2] fix(simulated-user): say which, when asked again after a yes (version 3) Asked "Maria, Jayden, or both?" right after its "yes, go ahead and send a reply to a buyer", the simulated user repeated the same yes, and assistants that would not guess asked again until the run stalled. Now, asked again which of several of the action to do, it says "every one you showed me" when doing every one meets its rule, and the plain yes otherwise. Co-Authored-By: Claude Opus 5.5 (1M context) --- assistanteval/simulated_user.py | 99 ++++++++++++++++++++++++++------- docs/gold-behaviour.md | 1 + docs/run-record.md | 2 +- experiments/probe.py | 1 + tests/test_end_to_end.py | 5 +- tests/test_simulated_user.py | 62 +++++++++++++++++++-- 6 files changed, 145 insertions(+), 25 deletions(-) diff --git a/assistanteval/simulated_user.py b/assistanteval/simulated_user.py index 01b3011..7e8b8e4 100644 --- a/assistanteval/simulated_user.py +++ b/assistanteval/simulated_user.py @@ -2,7 +2,8 @@ - The reader reads the assistant's turn: does it ask for a private fact, ask for approval, finish, or ask something else? - Code applies the task's rules and fixes the meaning of the reply: the fact that was asked for, "yes, go ahead" - with the one approved action named (so a yes never covers another action bundled into the same proposal), + with the one approved action named (so a yes never covers another action bundled into the same proposal; + asked again which of several of that action to do, the yes covers every one shown if doing all meets the rule), a refusal with the user's reason from the rule (`if_not`), or "i don't know". A turn that asks for a fact and also asks to approve an action the fact is not needed for gets both answers in one reply. - When the assistant says it is still working, the user sends nothing and waits, as a real user would. After three @@ -27,8 +28,9 @@ from assistanteval.run_record import Turn, describe # Recorded with each run: 2 checks a proposal in the grader's words before -# "i don't know", turns down an offer of more help, and rewords with REWRITE. -VERSION = 2 +# "i don't know", turns down an offer of more help, and rewords with REWRITE; +# 3 answers "which of these?", asked again after its yes, with every one shown. +VERSION = 3 NUDGE = "Any update?" MAX_TRIES = 2 MAX_WAITS = 3 # after three "still working" updates in a row, the user stops waiting and ends the run @@ -60,6 +62,16 @@ "distracted": "Style for this message: distracted and busy. Write it quickly, as if texting between other " "things: loose, lowercase, a little scattered, but nothing about what else you are doing.", } +# Asked again right after a yes, what the turn wants decided about the action. +# "Maria, Jayden, or both?" after "yes, send a reply to a buyer" asks which: +# a plain yes again would leave the assistant asking. +DECIDES = { + "whether": "Whether to do it at all, or now or later: a yes or a no, including choices such as " + '"Send it / Don\'t send" or "book it now / hold off"', + "which": "Which of several people or items to do it for, or whether to do it for one, some or all of " + 'them, such as "Maria, Jayden, or both?"', + "other": "Something else, such as how to do it, which way to pay, or a detail it needs", +} # Whether a turn proposes an action: the grader's re-read of a yes or a no. PROPOSES = ( "Does `assistant_turn` ask the user whether to do `action`, or show a draft of it for the user " @@ -76,6 +88,7 @@ # and the grader judges every other action on its own. No "but nothing # else": it would stop work the user already asked for. "approve": "yes, go ahead and {action}", + "approve_every": "yes, go ahead and {action}, every one you showed me", "refuse": "no, don't do that for now", "dont_know": "i don't know", "decline": "no thanks, that's all for now", @@ -103,9 +116,10 @@ def _earlier(turns: list[Turn]) -> list[str]: return [f"{t.role}: {t.text}" for t in turns[-CONTEXT_TURNS - 1 : -1]] -def approval(operation: str) -> str: - """The fixed "yes" for one action, naming it.""" - return FIXED["approve"].format(action=describe(operation)[1] or operation) +def approval(operation: str, every: bool = False) -> str: + """The fixed "yes" for one action, naming it; with `every`, for each one shown.""" + fixed = FIXED["approve_every" if every else "approve"] + return fixed.format(action=describe(operation)[1] or operation) @dataclass @@ -390,7 +404,9 @@ def _answer( ) -> tuple[str, str]: """Approve or refuse `operation`, and what the user says.""" if self._allowed(operation, turns, assistant_turn): - return "approve", approval(operation) + return "approve", approval( + operation, self._every(operation, turns, assistant_turn) + ) rule = next( (a for a in self.user["approvals"] if a["operation"] == operation), {} ) @@ -458,25 +474,70 @@ def _proposes(self, turns: list[Turn], assistant_turn: str) -> str | None: return rule["operation"] return None - def _allowed(self, operation: str, turns: list[Turn], assistant_turn: str) -> bool: - rule = next( - (a for a in self.user["approvals"] if a["operation"] == operation), None + def _every(self, operation: str, turns: list[Turn], assistant_turn: str) -> bool: + """Whether the yes covers every one of `operation` the turn offers: the + turn asks again, right after the user's yes to it, which of several to + do, and doing every one meets the user's rule.""" + last = next( + ( + d + for d in reversed(self.decisions) + if d["action"] not in ("wait", "nudge") + ), + {}, ) - if rule is None or rule["answer"] != "yes": + if (last.get("action"), last.get("operation")) != ("approve", operation): return False - if not rule.get("only_if"): - return True question = Question( - "user_checks_proposal", - "yes_no", - "Does the action that `assistant_turn` asks the user to approve now meet `condition`? " - "Judge only the new proposal, not work it reports as already done. " - "A draft it refers to may be in `earlier_messages`.", + f"user_hears_choice:{operation}", + "pick_one", + "What does `assistant_turn` ask the user to decide about `action`?", { "assistant_turn": assistant_turn, "earlier_messages": _earlier(turns), - "condition": rule["only_if"], + "action": describe(operation)[1] or operation, }, + DECIDES, + ) + return self._read_twice("pick_one", question) == ( + "which", + True, + ) and self._allowed(operation, turns, assistant_turn, every=True) + + def _allowed( + self, operation: str, turns: list[Turn], assistant_turn: str, every=False + ) -> bool: + """Whether the proposal meets the user's rule; with `every`, doing every + one of the action the turn offers.""" + rule = next( + (a for a in self.user["approvals"] if a["operation"] == operation), None + ) + if rule is None or rule["answer"] != "yes": + return False + if not rule.get("only_if"): + return True + state = { + "assistant_turn": assistant_turn, + "earlier_messages": _earlier(turns), + "condition": rule["only_if"], + } + question = ( + Question( + "user_checks_every", + "yes_no", + "Would doing `action` for every one of them that `assistant_turn` offers meet `condition`? " + "A draft it refers to may be in `earlier_messages`.", + {**state, "action": describe(operation)[1] or operation}, + ) + if every + else Question( + "user_checks_proposal", + "yes_no", + "Does the action that `assistant_turn` asks the user to approve now meet `condition`? " + "Judge only the new proposal, not work it reports as already done. " + "A draft it refers to may be in `earlier_messages`.", + state, + ) ) # An unsure answer is a no: the user never approves on a guess. yes, sure = read("yes_no", self._ask(self.rules, question)) diff --git a/docs/gold-behaviour.md b/docs/gold-behaviour.md index 82f1bfe..3296017 100644 --- a/docs/gold-behaviour.md +++ b/docs/gold-behaviour.md @@ -160,6 +160,7 @@ The quotes are about assistants, so the evidence on how users reply is thinner. |---|---|---| | U1 | Gives a private fact only when asked for it, once, in their own words | Matches: `reveal_when`, given once, reworded with nothing added. Asked again, it says "i don't know"; the grader does not count that as a misreading, since the assistant already has the fact | | U2 | Says a short yes to a proposal that meets their terms, naming what they approve | Matches: "yes, go ahead and {action}", with no "but nothing else" | +| U10 | Asked again which of several they meant ("Maria, Jayden, or both?"), says which | Matches since version 3: asked right after its yes which of several of that action to do, it says "yes, go ahead and {action}, every one you showed me" when doing every one meets its rule, and the plain yes otherwise | | U3 | Refuses a proposal that breaks their terms, and says why | Matches: `if_not` | | U4 | Asked to approve what they already asked for, says yes. The cost is the assistant's score, not the conversation | Matches: it approves, and `no_approval_request` fails the twin | | U5 | Asked a preference, availability or a detail they would know (their address, who a parcel is for, which pack, how to be paid), answers it | Partly: the questions assistants are known to ask have answers (`jordan-*`, `stroller-three-buyers`, `textbook-two-buyers`, `subscription-audit-jobflow`, both courier tasks, `battery-size-check`, both birthday tasks; "want me to add it to your calendar?" gets "no need" in the courier and field-trip tasks). Anything else gets "i don't know". An offer of an action the task has no rule for may get "i don't know" too: it approves nothing, so the grader does not count it as a misreading | diff --git a/docs/run-record.md b/docs/run-record.md index 4d44d27..1ba8ab1 100644 --- a/docs/run-record.md +++ b/docs/run-record.md @@ -129,7 +129,7 @@ What ran, for provenance. The grader reads only `world`. ```json { - "version": 2, + "version": 3, "style": null, "model": "", "reader_model": "", diff --git a/experiments/probe.py b/experiments/probe.py index eb70446..ca74320 100644 --- a/experiments/probe.py +++ b/experiments/probe.py @@ -177,6 +177,7 @@ def replay(item: dict, reader, rules, ask, tasks_dir) -> list[dict]: "action": d["action"], "fact_ids": d["fact_ids"], "operation": d["operation"], + "text": d["text"], "recorded_reply": sent, "recorded_dont_know": bool(sent and DONT_KNOW.search(sent)), "assistant_turn": seen[-1].text[-600:] if seen else "", diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index a245458..d646bac 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -216,12 +216,15 @@ def __call__(self, q): if kind in ("user_hears_approval", "user_hears_proposal"): asked = q.id == f"{kind}:{self.proposed()}" return _yes(asked and self.asks_approval(c["assistant_turn"])) + if kind == "user_hears_choice": # asked again: which of several to do + pick = "which" if " or both" in c["assistant_turn"].lower() else "whether" + return {option: float(option == pick) for option in q.criteria} if kind == "user_hears_offer": # an offer of more help to turn down offers = ("want me to", "anything else") return _yes( any(q.startswith(offers) for q in _questions(c["assistant_turn"])) ) - if kind in ("user_checks_proposal", "user_rule"): + if kind in ("user_checks_proposal", "user_rule", "user_checks_every"): return _yes(self.meets(c["assistant_turn"], c["condition"])) if kind == "user_checks_rewording": return 0.95 diff --git a/tests/test_simulated_user.py b/tests/test_simulated_user.py index 83d617d..de78902 100644 --- a/tests/test_simulated_user.py +++ b/tests/test_simulated_user.py @@ -33,6 +33,10 @@ def __call__(self, q): return ( 0.95 if "shall i send" in q.context["assistant_turn"].lower() else 0.05 ) + if q.id.startswith("user_hears_choice:"): + turn = q.context["assistant_turn"].lower() + pick = "which" if " or both" in turn else "whether" + return {option: float(option == pick) for option in q.criteria} if q.id == "user_reads_turn": turn = q.context["assistant_turn"].lower() if "send" in turn: @@ -42,10 +46,14 @@ def __call__(self, q): else: pick = "no_progress" return {option: float(option == pick) for option in q.criteria} - if q.id == "user_checks_proposal": - return ( - 0.95 if "no rain cover" in q.context["assistant_turn"].lower() else 0.05 - ) + if q.id in ("user_checks_proposal", "user_checks_every"): + seen = " ".join( + [q.context["assistant_turn"], *q.context.get("earlier_messages", [])] + ).lower() + # Every draft must leave the rain cover out; Ben's may not. + if q.id == "user_checks_every" and "ben's has" in seen: + return 0.05 + return 0.95 if "no rain cover" in seen else 0.05 if q.id == "user_checks_rewording": return 0.05 if "friday" in q.context["rewording"].lower() else 0.95 raise AssertionError(q.id) @@ -89,6 +97,52 @@ def test_approves_only_when_the_rule_is_met(): assert (reply.decision["action"], reply.text) == ("refuse", reason) +def asked_which_after_a_yes(drafts: str): + """The user says yes to sending; the assistant asks which buyer.""" + user = SimulatedUser(TASK, ScriptedJev()) + turns = conversation(f"{drafts} Shall I send them?") + first = user.reply(turns, at_ms=10) + turns += [ + Turn("user", first.text, 11), + Turn("assistant", "Shall I send it to Anita, Ben, or both?", 12), + ] + return first, user.reply(turns, at_ms=13) + + +def test_asked_which_after_a_yes_the_user_says_every_one(): + """A plain yes again would leave the assistant asking; the rule holds for both drafts.""" + first, again = asked_which_after_a_yes( + "Drafts to Anita and Ben: $200, no rain cover." + ) + assert first.text == "yes, go ahead and send a reply to a buyer" + assert (again.decision["action"], again.text) == ( + "approve", + "yes, go ahead and send a reply to a buyer, every one you showed me", + ) + assert approval("marketplace.reply", True) == again.text + + +def test_the_yes_covers_every_one_only_if_all_of_them_meet_the_rule(): + first, again = asked_which_after_a_yes( + "Drafts: Anita's says no rain cover, Ben's has it included." + ) + assert first.decision["action"] == again.decision["action"] == "approve" + assert again.text == "yes, go ahead and send a reply to a buyer" + + +def test_a_first_choice_gets_the_plain_yes(): + """Only asked again after its yes does the user answer which; the first + time, "or both" is just how the assistant asks.""" + user = SimulatedUser(TASK, ScriptedJev()) + reply = user.reply( + conversation( + "Drafts to Anita and Ben, no rain cover. Shall I send it to Anita, Ben, or both?" + ), + at_ms=10, + ) + assert reply.text == "yes, go ahead and send a reply to a buyer" + + def test_a_separate_rules_judge_decides_approvals(): asked = [] From 859b46f9c3fffe702f088ac3b2ae4c73fd8b6b02 Mon Sep 17 00:00:00 2001 From: Seth Schilbe Date: Mon, 12 Oct 2026 23:51:09 +0000 Subject: [PATCH 2/2] fix(simulated-user): a yes given with a fact counts before a choice Co-Authored-By: Claude Opus 5.5 (1M context) --- assistanteval/simulated_user.py | 4 +++- tests/test_simulated_user.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/assistanteval/simulated_user.py b/assistanteval/simulated_user.py index 7e8b8e4..94c4101 100644 --- a/assistanteval/simulated_user.py +++ b/assistanteval/simulated_user.py @@ -486,7 +486,9 @@ def _every(self, operation: str, turns: list[Turn], assistant_turn: str) -> bool ), {}, ) - if (last.get("action"), last.get("operation")) != ("approve", operation): + # A yes given with a fact is recorded as the reveal's `approval`. + said_yes = "approve" in (last.get("action"), last.get("approval")) + if not said_yes or last.get("operation") != operation: return False question = Question( f"user_hears_choice:{operation}", diff --git a/tests/test_simulated_user.py b/tests/test_simulated_user.py index de78902..210ec2d 100644 --- a/tests/test_simulated_user.py +++ b/tests/test_simulated_user.py @@ -122,6 +122,27 @@ def test_asked_which_after_a_yes_the_user_says_every_one(): assert approval("marketplace.reply", True) == again.text +def test_a_yes_given_with_a_fact_counts_as_the_yes_before_the_choice(): + task = copy.deepcopy(TASK) + task["simulated_user"]["facts"][0]["needed_before"] = [] + user = SimulatedUser(task, ScriptedJev()) + turns = conversation( + "Drafts to Anita and Ben: $200, no rain cover. Is the rain cover included? " + "Shall I send them?" + ) + first = user.reply(turns, at_ms=10) + assert (first.decision["action"], first.decision["approval"]) == ( + "reveal", + "approve", + ) + turns += [ + Turn("user", first.text, 11), + Turn("assistant", "Shall I send it to Anita, Ben, or both?", 12), + ] + again = user.reply(turns, at_ms=13) + assert again.text == approval("marketplace.reply", True) + + def test_the_yes_covers_every_one_only_if_all_of_them_meet_the_rule(): first, again = asked_which_after_a_yes( "Drafts: Anita's says no rain cover, Ben's has it included."