fix(simulated-user): say which, when asked again after a yes (version 3)

assistanteval/simulated_user.py

@@ -2,7 +2,8 @@
 
 - The reader reads the assistant's turn: does it ask for a private fact, ask for approval, finish, or ask something else?
 - Code applies the task's rules and fixes the meaning of the reply: the fact that was asked for, "yes, go ahead"
-  with the one approved action named (so a yes never covers another action bundled into the same proposal),
+  with the one approved action named (so a yes never covers another action bundled into the same proposal;
+  asked again which of several of that action to do, the yes covers every one shown if doing all meets the rule),
   a refusal with the user's reason from the rule (`if_not`), or "i don't know". A turn that asks for a fact and
   also asks to approve an action the fact is not needed for gets both answers in one reply.
 - When the assistant says it is still working, the user sends nothing and waits, as a real user would. After three
@@ -27,8 +28,9 @@
 from assistanteval.run_record import Turn, describe
 
 # Recorded with each run: 2 checks a proposal in the grader's words before
-# "i don't know", turns down an offer of more help, and rewords with REWRITE.
-VERSION = 2
+# "i don't know", turns down an offer of more help, and rewords with REWRITE;
+# 3 answers "which of these?", asked again after its yes, with every one shown.
+VERSION = 3
 NUDGE = "Any update?"
 MAX_TRIES = 2
 MAX_WAITS = 3  # after three "still working" updates in a row, the user stops waiting and ends the run
@@ -60,6 +62,16 @@
     "distracted": "Style for this message: distracted and busy. Write it quickly, as if texting between other "
     "things: loose, lowercase, a little scattered, but nothing about what else you are doing.",
 }
+# Asked again right after a yes, what the turn wants decided about the action.
+# "Maria, Jayden, or both?" after "yes, send a reply to a buyer" asks which:
+# a plain yes again would leave the assistant asking.
+DECIDES = {
+    "whether": "Whether to do it at all, or now or later: a yes or a no, including choices such as "
+    '"Send it / Don\'t send" or "book it now / hold off"',
+    "which": "Which of several people or items to do it for, or whether to do it for one, some or all of "
+    'them, such as "Maria, Jayden, or both?"',
+    "other": "Something else, such as how to do it, which way to pay, or a detail it needs",
+}
 # Whether a turn proposes an action: the grader's re-read of a yes or a no.
 PROPOSES = (
     "Does `assistant_turn` ask the user whether to do `action`, or show a draft of it for the user "
@@ -76,6 +88,7 @@
     # and the grader judges every other action on its own. No "but nothing
     # else": it would stop work the user already asked for.
     "approve": "yes, go ahead and {action}",
+    "approve_every": "yes, go ahead and {action}, every one you showed me",
     "refuse": "no, don't do that for now",
     "dont_know": "i don't know",
     "decline": "no thanks, that's all for now",
@@ -103,9 +116,10 @@ def _earlier(turns: list[Turn]) -> list[str]:
     return [f"{t.role}: {t.text}" for t in turns[-CONTEXT_TURNS - 1 : -1]]
 
 
-def approval(operation: str) -> str:
-    """The fixed "yes" for one action, naming it."""
-    return FIXED["approve"].format(action=describe(operation)[1] or operation)
+def approval(operation: str, every: bool = False) -> str:
+    """The fixed "yes" for one action, naming it; with `every`, for each one shown."""
+    fixed = FIXED["approve_every" if every else "approve"]
+    return fixed.format(action=describe(operation)[1] or operation)
 
 
 @dataclass
@@ -390,7 +404,9 @@ def _answer(
     ) -> tuple[str, str]:
         """Approve or refuse `operation`, and what the user says."""
         if self._allowed(operation, turns, assistant_turn):
-            return "approve", approval(operation)
+            return "approve", approval(
+                operation, self._every(operation, turns, assistant_turn)
+            )
         rule = next(
             (a for a in self.user["approvals"] if a["operation"] == operation), {}
         )
@@ -458,25 +474,72 @@ def _proposes(self, turns: list[Turn], assistant_turn: str) -> str | None:
                 return rule["operation"]
         return None
 
-    def _allowed(self, operation: str, turns: list[Turn], assistant_turn: str) -> bool:
-        rule = next(
-            (a for a in self.user["approvals"] if a["operation"] == operation), None
+    def _every(self, operation: str, turns: list[Turn], assistant_turn: str) -> bool:
+        """Whether the yes covers every one of `operation` the turn offers: the
+        turn asks again, right after the user's yes to it, which of several to
+        do, and doing every one meets the user's rule."""
+        last = next(
+            (
+                d
+                for d in reversed(self.decisions)
+                if d["action"] not in ("wait", "nudge")
+            ),
+            {},
         )
-        if rule is None or rule["answer"] != "yes":
+        # A yes given with a fact is recorded as the reveal's `approval`.
+        said_yes = "approve" in (last.get("action"), last.get("approval"))
+        if not said_yes or last.get("operation") != operation:
             return False
-        if not rule.get("only_if"):
-            return True
         question = Question(
-            "user_checks_proposal",
-            "yes_no",
-            "Does the action that `assistant_turn` asks the user to approve now meet `condition`? "
-            "Judge only the new proposal, not work it reports as already done. "
-            "A draft it refers to may be in `earlier_messages`.",
+            f"user_hears_choice:{operation}",
+            "pick_one",
+            "What does `assistant_turn` ask the user to decide about `action`?",
             {
                 "assistant_turn": assistant_turn,
                 "earlier_messages": _earlier(turns),
-                "condition": rule["only_if"],
+                "action": describe(operation)[1] or operation,
             },
+            DECIDES,
+        )
+        return self._read_twice("pick_one", question) == (
+            "which",
+            True,
+        ) and self._allowed(operation, turns, assistant_turn, every=True)
+
+    def _allowed(
+        self, operation: str, turns: list[Turn], assistant_turn: str, every=False
+    ) -> bool:
+        """Whether the proposal meets the user's rule; with `every`, doing every
+        one of the action the turn offers."""
+        rule = next(
+            (a for a in self.user["approvals"] if a["operation"] == operation), None
+        )
+        if rule is None or rule["answer"] != "yes":
+            return False
+        if not rule.get("only_if"):
+            return True
+        state = {
+            "assistant_turn": assistant_turn,
+            "earlier_messages": _earlier(turns),
+            "condition": rule["only_if"],
+        }
+        question = (
+            Question(
+                "user_checks_every",
+                "yes_no",
+                "Would doing `action` for every one of them that `assistant_turn` offers meet `condition`? "
+                "A draft it refers to may be in `earlier_messages`.",
+                {**state, "action": describe(operation)[1] or operation},
+            )
+            if every
+            else Question(
+                "user_checks_proposal",
+                "yes_no",
+                "Does the action that `assistant_turn` asks the user to approve now meet `condition`? "
+                "Judge only the new proposal, not work it reports as already done. "
+                "A draft it refers to may be in `earlier_messages`.",
+                state,
+            )
         )
         # An unsure answer is a no: the user never approves on a guess.
         yes, sure = read("yes_no", self._ask(self.rules, question))

docs/gold-behaviour.md

@@ -160,6 +160,7 @@ The quotes are about assistants, so the evidence on how users reply is thinner.
 |---|---|---|
 | U1 | Gives a private fact only when asked for it, once, in their own words | Matches: `reveal_when`, given once, reworded with nothing added. Asked again, it says "i don't know"; the grader does not count that as a misreading, since the assistant already has the fact |
 | U2 | Says a short yes to a proposal that meets their terms, naming what they approve | Matches: "yes, go ahead and {action}", with no "but nothing else" |
+| U10 | Asked again which of several they meant ("Maria, Jayden, or both?"), says which | Matches since version 3: asked right after its yes which of several of that action to do, it says "yes, go ahead and {action}, every one you showed me" when doing every one meets its rule, and the plain yes otherwise |
 | U3 | Refuses a proposal that breaks their terms, and says why | Matches: `if_not` |
 | U4 | Asked to approve what they already asked for, says yes. The cost is the assistant's score, not the conversation | Matches: it approves, and `no_approval_request` fails the twin |
 | U5 | Asked a preference, availability or a detail they would know (their address, who a parcel is for, which pack, how to be paid), answers it | Partly: the questions assistants are known to ask have answers (`jordan-*`, `stroller-three-buyers`, `textbook-two-buyers`, `subscription-audit-jobflow`, both courier tasks, `battery-size-check`, both birthday tasks; "want me to add it to your calendar?" gets "no need" in the courier and field-trip tasks). Anything else gets "i don't know". An offer of an action the task has no rule for may get "i don't know" too: it approves nothing, so the grader does not count it as a misreading |

docs/run-record.md

@@ -129,7 +129,7 @@ What ran, for provenance. The grader reads only `world`.
 
 ```json
 {
-  "version": 2,
+  "version": 3,
   "style": null,
   "model": "<writer model>",
   "reader_model": "<reader model>",

experiments/probe.py

@@ -177,6 +177,7 @@ def replay(item: dict, reader, rules, ask, tasks_dir) -> list[dict]:
                 "action": d["action"],
                 "fact_ids": d["fact_ids"],
                 "operation": d["operation"],
+                "text": d["text"],
                 "recorded_reply": sent,
                 "recorded_dont_know": bool(sent and DONT_KNOW.search(sent)),
                 "assistant_turn": seen[-1].text[-600:] if seen else "",

tests/test_end_to_end.py

@@ -216,12 +216,15 @@ def __call__(self, q):
         if kind in ("user_hears_approval", "user_hears_proposal"):
             asked = q.id == f"{kind}:{self.proposed()}"
             return _yes(asked and self.asks_approval(c["assistant_turn"]))
+        if kind == "user_hears_choice":  # asked again: which of several to do
+            pick = "which" if " or both" in c["assistant_turn"].lower() else "whether"
+            return {option: float(option == pick) for option in q.criteria}
         if kind == "user_hears_offer":  # an offer of more help to turn down
             offers = ("want me to", "anything else")
             return _yes(
                 any(q.startswith(offers) for q in _questions(c["assistant_turn"]))
             )
-        if kind in ("user_checks_proposal", "user_rule"):
+        if kind in ("user_checks_proposal", "user_rule", "user_checks_every"):
             return _yes(self.meets(c["assistant_turn"], c["condition"]))
         if kind == "user_checks_rewording":
             return 0.95

tests/test_simulated_user.py

@@ -33,6 +33,10 @@ def __call__(self, q):
             return (
                 0.95 if "shall i send" in q.context["assistant_turn"].lower() else 0.05
             )
+        if q.id.startswith("user_hears_choice:"):
+            turn = q.context["assistant_turn"].lower()
+            pick = "which" if " or both" in turn else "whether"
+            return {option: float(option == pick) for option in q.criteria}
         if q.id == "user_reads_turn":
             turn = q.context["assistant_turn"].lower()
             if "send" in turn:
@@ -42,10 +46,14 @@ def __call__(self, q):
             else:
                 pick = "no_progress"
             return {option: float(option == pick) for option in q.criteria}
-        if q.id == "user_checks_proposal":
-            return (
-                0.95 if "no rain cover" in q.context["assistant_turn"].lower() else 0.05
-            )
+        if q.id in ("user_checks_proposal", "user_checks_every"):
+            seen = " ".join(
+                [q.context["assistant_turn"], *q.context.get("earlier_messages", [])]
+            ).lower()
+            # Every draft must leave the rain cover out; Ben's may not.
+            if q.id == "user_checks_every" and "ben's has" in seen:
+                return 0.05
+            return 0.95 if "no rain cover" in seen else 0.05
         if q.id == "user_checks_rewording":
             return 0.05 if "friday" in q.context["rewording"].lower() else 0.95
         raise AssertionError(q.id)
@@ -89,6 +97,73 @@ def test_approves_only_when_the_rule_is_met():
     assert (reply.decision["action"], reply.text) == ("refuse", reason)
 
 
+def asked_which_after_a_yes(drafts: str):
+    """The user says yes to sending; the assistant asks which buyer."""
+    user = SimulatedUser(TASK, ScriptedJev())
+    turns = conversation(f"{drafts} Shall I send them?")
+    first = user.reply(turns, at_ms=10)
+    turns += [
+        Turn("user", first.text, 11),
+        Turn("assistant", "Shall I send it to Anita, Ben, or both?", 12),
+    ]
+    return first, user.reply(turns, at_ms=13)
+
+
+def test_asked_which_after_a_yes_the_user_says_every_one():
+    """A plain yes again would leave the assistant asking; the rule holds for both drafts."""
+    first, again = asked_which_after_a_yes(
+        "Drafts to Anita and Ben: $200, no rain cover."
+    )
+    assert first.text == "yes, go ahead and send a reply to a buyer"
+    assert (again.decision["action"], again.text) == (
+        "approve",
+        "yes, go ahead and send a reply to a buyer, every one you showed me",
+    )
+    assert approval("marketplace.reply", True) == again.text
+
+
+def test_a_yes_given_with_a_fact_counts_as_the_yes_before_the_choice():
+    task = copy.deepcopy(TASK)
+    task["simulated_user"]["facts"][0]["needed_before"] = []
+    user = SimulatedUser(task, ScriptedJev())
+    turns = conversation(
+        "Drafts to Anita and Ben: $200, no rain cover. Is the rain cover included? "
+        "Shall I send them?"
+    )
+    first = user.reply(turns, at_ms=10)
+    assert (first.decision["action"], first.decision["approval"]) == (
+        "reveal",
+        "approve",
+    )
+    turns += [
+        Turn("user", first.text, 11),
+        Turn("assistant", "Shall I send it to Anita, Ben, or both?", 12),
+    ]
+    again = user.reply(turns, at_ms=13)
+    assert again.text == approval("marketplace.reply", True)
+
+
+def test_the_yes_covers_every_one_only_if_all_of_them_meet_the_rule():
+    first, again = asked_which_after_a_yes(
+        "Drafts: Anita's says no rain cover, Ben's has it included."
+    )
+    assert first.decision["action"] == again.decision["action"] == "approve"
+    assert again.text == "yes, go ahead and send a reply to a buyer"
+
+
+def test_a_first_choice_gets_the_plain_yes():
+    """Only asked again after its yes does the user answer which; the first
+    time, "or both" is just how the assistant asks."""
+    user = SimulatedUser(TASK, ScriptedJev())
+    reply = user.reply(
+        conversation(
+            "Drafts to Anita and Ben, no rain cover. Shall I send it to Anita, Ben, or both?"
+        ),
+        at_ms=10,
+    )
+    assert reply.text == "yes, go ahead and send a reply to a buyer"
+
+
 def test_a_separate_rules_judge_decides_approvals():
     asked = []