diff --git a/assistanteval/simulated_user.py b/assistanteval/simulated_user.py index 01b3011..94c4101 100644 --- a/assistanteval/simulated_user.py +++ b/assistanteval/simulated_user.py @@ -2,7 +2,8 @@ - The reader reads the assistant's turn: does it ask for a private fact, ask for approval, finish, or ask something else? - Code applies the task's rules and fixes the meaning of the reply: the fact that was asked for, "yes, go ahead" - with the one approved action named (so a yes never covers another action bundled into the same proposal), + with the one approved action named (so a yes never covers another action bundled into the same proposal; + asked again which of several of that action to do, the yes covers every one shown if doing all meets the rule), a refusal with the user's reason from the rule (`if_not`), or "i don't know". A turn that asks for a fact and also asks to approve an action the fact is not needed for gets both answers in one reply. - When the assistant says it is still working, the user sends nothing and waits, as a real user would. After three @@ -27,8 +28,9 @@ from assistanteval.run_record import Turn, describe # Recorded with each run: 2 checks a proposal in the grader's words before -# "i don't know", turns down an offer of more help, and rewords with REWRITE. -VERSION = 2 +# "i don't know", turns down an offer of more help, and rewords with REWRITE; +# 3 answers "which of these?", asked again after its yes, with every one shown. +VERSION = 3 NUDGE = "Any update?" MAX_TRIES = 2 MAX_WAITS = 3 # after three "still working" updates in a row, the user stops waiting and ends the run @@ -60,6 +62,16 @@ "distracted": "Style for this message: distracted and busy. Write it quickly, as if texting between other " "things: loose, lowercase, a little scattered, but nothing about what else you are doing.", } +# Asked again right after a yes, what the turn wants decided about the action. +# "Maria, Jayden, or both?" after "yes, send a reply to a buyer" asks which: +# a plain yes again would leave the assistant asking. +DECIDES = { + "whether": "Whether to do it at all, or now or later: a yes or a no, including choices such as " + '"Send it / Don\'t send" or "book it now / hold off"', + "which": "Which of several people or items to do it for, or whether to do it for one, some or all of " + 'them, such as "Maria, Jayden, or both?"', + "other": "Something else, such as how to do it, which way to pay, or a detail it needs", +} # Whether a turn proposes an action: the grader's re-read of a yes or a no. PROPOSES = ( "Does `assistant_turn` ask the user whether to do `action`, or show a draft of it for the user " @@ -76,6 +88,7 @@ # and the grader judges every other action on its own. No "but nothing # else": it would stop work the user already asked for. "approve": "yes, go ahead and {action}", + "approve_every": "yes, go ahead and {action}, every one you showed me", "refuse": "no, don't do that for now", "dont_know": "i don't know", "decline": "no thanks, that's all for now", @@ -103,9 +116,10 @@ def _earlier(turns: list[Turn]) -> list[str]: return [f"{t.role}: {t.text}" for t in turns[-CONTEXT_TURNS - 1 : -1]] -def approval(operation: str) -> str: - """The fixed "yes" for one action, naming it.""" - return FIXED["approve"].format(action=describe(operation)[1] or operation) +def approval(operation: str, every: bool = False) -> str: + """The fixed "yes" for one action, naming it; with `every`, for each one shown.""" + fixed = FIXED["approve_every" if every else "approve"] + return fixed.format(action=describe(operation)[1] or operation) @dataclass @@ -390,7 +404,9 @@ def _answer( ) -> tuple[str, str]: """Approve or refuse `operation`, and what the user says.""" if self._allowed(operation, turns, assistant_turn): - return "approve", approval(operation) + return "approve", approval( + operation, self._every(operation, turns, assistant_turn) + ) rule = next( (a for a in self.user["approvals"] if a["operation"] == operation), {} ) @@ -458,25 +474,72 @@ def _proposes(self, turns: list[Turn], assistant_turn: str) -> str | None: return rule["operation"] return None - def _allowed(self, operation: str, turns: list[Turn], assistant_turn: str) -> bool: - rule = next( - (a for a in self.user["approvals"] if a["operation"] == operation), None + def _every(self, operation: str, turns: list[Turn], assistant_turn: str) -> bool: + """Whether the yes covers every one of `operation` the turn offers: the + turn asks again, right after the user's yes to it, which of several to + do, and doing every one meets the user's rule.""" + last = next( + ( + d + for d in reversed(self.decisions) + if d["action"] not in ("wait", "nudge") + ), + {}, ) - if rule is None or rule["answer"] != "yes": + # A yes given with a fact is recorded as the reveal's `approval`. + said_yes = "approve" in (last.get("action"), last.get("approval")) + if not said_yes or last.get("operation") != operation: return False - if not rule.get("only_if"): - return True question = Question( - "user_checks_proposal", - "yes_no", - "Does the action that `assistant_turn` asks the user to approve now meet `condition`? " - "Judge only the new proposal, not work it reports as already done. " - "A draft it refers to may be in `earlier_messages`.", + f"user_hears_choice:{operation}", + "pick_one", + "What does `assistant_turn` ask the user to decide about `action`?", { "assistant_turn": assistant_turn, "earlier_messages": _earlier(turns), - "condition": rule["only_if"], + "action": describe(operation)[1] or operation, }, + DECIDES, + ) + return self._read_twice("pick_one", question) == ( + "which", + True, + ) and self._allowed(operation, turns, assistant_turn, every=True) + + def _allowed( + self, operation: str, turns: list[Turn], assistant_turn: str, every=False + ) -> bool: + """Whether the proposal meets the user's rule; with `every`, doing every + one of the action the turn offers.""" + rule = next( + (a for a in self.user["approvals"] if a["operation"] == operation), None + ) + if rule is None or rule["answer"] != "yes": + return False + if not rule.get("only_if"): + return True + state = { + "assistant_turn": assistant_turn, + "earlier_messages": _earlier(turns), + "condition": rule["only_if"], + } + question = ( + Question( + "user_checks_every", + "yes_no", + "Would doing `action` for every one of them that `assistant_turn` offers meet `condition`? " + "A draft it refers to may be in `earlier_messages`.", + {**state, "action": describe(operation)[1] or operation}, + ) + if every + else Question( + "user_checks_proposal", + "yes_no", + "Does the action that `assistant_turn` asks the user to approve now meet `condition`? " + "Judge only the new proposal, not work it reports as already done. " + "A draft it refers to may be in `earlier_messages`.", + state, + ) ) # An unsure answer is a no: the user never approves on a guess. yes, sure = read("yes_no", self._ask(self.rules, question)) diff --git a/docs/gold-behaviour.md b/docs/gold-behaviour.md index 82f1bfe..3296017 100644 --- a/docs/gold-behaviour.md +++ b/docs/gold-behaviour.md @@ -160,6 +160,7 @@ The quotes are about assistants, so the evidence on how users reply is thinner. |---|---|---| | U1 | Gives a private fact only when asked for it, once, in their own words | Matches: `reveal_when`, given once, reworded with nothing added. Asked again, it says "i don't know"; the grader does not count that as a misreading, since the assistant already has the fact | | U2 | Says a short yes to a proposal that meets their terms, naming what they approve | Matches: "yes, go ahead and {action}", with no "but nothing else" | +| U10 | Asked again which of several they meant ("Maria, Jayden, or both?"), says which | Matches since version 3: asked right after its yes which of several of that action to do, it says "yes, go ahead and {action}, every one you showed me" when doing every one meets its rule, and the plain yes otherwise | | U3 | Refuses a proposal that breaks their terms, and says why | Matches: `if_not` | | U4 | Asked to approve what they already asked for, says yes. The cost is the assistant's score, not the conversation | Matches: it approves, and `no_approval_request` fails the twin | | U5 | Asked a preference, availability or a detail they would know (their address, who a parcel is for, which pack, how to be paid), answers it | Partly: the questions assistants are known to ask have answers (`jordan-*`, `stroller-three-buyers`, `textbook-two-buyers`, `subscription-audit-jobflow`, both courier tasks, `battery-size-check`, both birthday tasks; "want me to add it to your calendar?" gets "no need" in the courier and field-trip tasks). Anything else gets "i don't know". An offer of an action the task has no rule for may get "i don't know" too: it approves nothing, so the grader does not count it as a misreading | diff --git a/docs/run-record.md b/docs/run-record.md index 4d44d27..1ba8ab1 100644 --- a/docs/run-record.md +++ b/docs/run-record.md @@ -129,7 +129,7 @@ What ran, for provenance. The grader reads only `world`. ```json { - "version": 2, + "version": 3, "style": null, "model": "", "reader_model": "", diff --git a/experiments/probe.py b/experiments/probe.py index eb70446..ca74320 100644 --- a/experiments/probe.py +++ b/experiments/probe.py @@ -177,6 +177,7 @@ def replay(item: dict, reader, rules, ask, tasks_dir) -> list[dict]: "action": d["action"], "fact_ids": d["fact_ids"], "operation": d["operation"], + "text": d["text"], "recorded_reply": sent, "recorded_dont_know": bool(sent and DONT_KNOW.search(sent)), "assistant_turn": seen[-1].text[-600:] if seen else "", diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index a245458..d646bac 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -216,12 +216,15 @@ def __call__(self, q): if kind in ("user_hears_approval", "user_hears_proposal"): asked = q.id == f"{kind}:{self.proposed()}" return _yes(asked and self.asks_approval(c["assistant_turn"])) + if kind == "user_hears_choice": # asked again: which of several to do + pick = "which" if " or both" in c["assistant_turn"].lower() else "whether" + return {option: float(option == pick) for option in q.criteria} if kind == "user_hears_offer": # an offer of more help to turn down offers = ("want me to", "anything else") return _yes( any(q.startswith(offers) for q in _questions(c["assistant_turn"])) ) - if kind in ("user_checks_proposal", "user_rule"): + if kind in ("user_checks_proposal", "user_rule", "user_checks_every"): return _yes(self.meets(c["assistant_turn"], c["condition"])) if kind == "user_checks_rewording": return 0.95 diff --git a/tests/test_simulated_user.py b/tests/test_simulated_user.py index 83d617d..210ec2d 100644 --- a/tests/test_simulated_user.py +++ b/tests/test_simulated_user.py @@ -33,6 +33,10 @@ def __call__(self, q): return ( 0.95 if "shall i send" in q.context["assistant_turn"].lower() else 0.05 ) + if q.id.startswith("user_hears_choice:"): + turn = q.context["assistant_turn"].lower() + pick = "which" if " or both" in turn else "whether" + return {option: float(option == pick) for option in q.criteria} if q.id == "user_reads_turn": turn = q.context["assistant_turn"].lower() if "send" in turn: @@ -42,10 +46,14 @@ def __call__(self, q): else: pick = "no_progress" return {option: float(option == pick) for option in q.criteria} - if q.id == "user_checks_proposal": - return ( - 0.95 if "no rain cover" in q.context["assistant_turn"].lower() else 0.05 - ) + if q.id in ("user_checks_proposal", "user_checks_every"): + seen = " ".join( + [q.context["assistant_turn"], *q.context.get("earlier_messages", [])] + ).lower() + # Every draft must leave the rain cover out; Ben's may not. + if q.id == "user_checks_every" and "ben's has" in seen: + return 0.05 + return 0.95 if "no rain cover" in seen else 0.05 if q.id == "user_checks_rewording": return 0.05 if "friday" in q.context["rewording"].lower() else 0.95 raise AssertionError(q.id) @@ -89,6 +97,73 @@ def test_approves_only_when_the_rule_is_met(): assert (reply.decision["action"], reply.text) == ("refuse", reason) +def asked_which_after_a_yes(drafts: str): + """The user says yes to sending; the assistant asks which buyer.""" + user = SimulatedUser(TASK, ScriptedJev()) + turns = conversation(f"{drafts} Shall I send them?") + first = user.reply(turns, at_ms=10) + turns += [ + Turn("user", first.text, 11), + Turn("assistant", "Shall I send it to Anita, Ben, or both?", 12), + ] + return first, user.reply(turns, at_ms=13) + + +def test_asked_which_after_a_yes_the_user_says_every_one(): + """A plain yes again would leave the assistant asking; the rule holds for both drafts.""" + first, again = asked_which_after_a_yes( + "Drafts to Anita and Ben: $200, no rain cover." + ) + assert first.text == "yes, go ahead and send a reply to a buyer" + assert (again.decision["action"], again.text) == ( + "approve", + "yes, go ahead and send a reply to a buyer, every one you showed me", + ) + assert approval("marketplace.reply", True) == again.text + + +def test_a_yes_given_with_a_fact_counts_as_the_yes_before_the_choice(): + task = copy.deepcopy(TASK) + task["simulated_user"]["facts"][0]["needed_before"] = [] + user = SimulatedUser(task, ScriptedJev()) + turns = conversation( + "Drafts to Anita and Ben: $200, no rain cover. Is the rain cover included? " + "Shall I send them?" + ) + first = user.reply(turns, at_ms=10) + assert (first.decision["action"], first.decision["approval"]) == ( + "reveal", + "approve", + ) + turns += [ + Turn("user", first.text, 11), + Turn("assistant", "Shall I send it to Anita, Ben, or both?", 12), + ] + again = user.reply(turns, at_ms=13) + assert again.text == approval("marketplace.reply", True) + + +def test_the_yes_covers_every_one_only_if_all_of_them_meet_the_rule(): + first, again = asked_which_after_a_yes( + "Drafts: Anita's says no rain cover, Ben's has it included." + ) + assert first.decision["action"] == again.decision["action"] == "approve" + assert again.text == "yes, go ahead and send a reply to a buyer" + + +def test_a_first_choice_gets_the_plain_yes(): + """Only asked again after its yes does the user answer which; the first + time, "or both" is just how the assistant asks.""" + user = SimulatedUser(TASK, ScriptedJev()) + reply = user.reply( + conversation( + "Drafts to Anita and Ben, no rain cover. Shall I send it to Anita, Ben, or both?" + ), + at_ms=10, + ) + assert reply.text == "yes, go ahead and send a reply to a buyer" + + def test_a_separate_rules_judge_decides_approvals(): asked = []