From 61c805f399710e19f0e090a09ff59a9a93e6bde3 Mon Sep 17 00:00:00 2001 From: isarasua Date: Sat, 10 Oct 2026 18:00:51 +0200 Subject: [PATCH] refactor(engine)!: remove the never-called should_continue_after_turn hook Nothing in the simulation loop calls `should_continue_after_turn`, so it never stopped a conversation. `safety_agentic`, the only probe that set it (through `AgenticMixin`), stops through its own loop. But `AgenticMixin`, the probe authoring guide and the agentic template still presented it as a working stop, so a new probe that relied on it would never stop where it meant to. It is removed from `BaseProbe`, `AgenticMixin`, the guide and the template, along with the tests that exercised only the hook. No behaviour changes. A probe outside this repository that calls `super().should_continue_after_turn()` would now fail; one that only overrides it is unaffected. Co-Authored-By: Claude Opus 5.5 Signed-off-by: isarasua --- docs/engine/AUTHORING_A_PROBE.md | 1 - src/usersim/engine/core/probes.py | 23 +------------------- templates/probe/generator_agentic.py | 5 ++--- tests/engine/core/test_probes_substrate.py | 25 ---------------------- 4 files changed, 3 insertions(+), 51 deletions(-) diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md index 0e2fc61..18d3fed 100644 --- a/docs/engine/AUTHORING_A_PROBE.md +++ b/docs/engine/AUTHORING_A_PROBE.md @@ -143,7 +143,6 @@ implementations. | `should_succeed(state)` | no | `True` | Always: assert your probe's join key is present | | `should_inline_judge_user_turn(turn_idx, state)` | no | `True` | Rarely: `AgenticMixin` skips turn-0 | | `should_inline_judge_assistant_turn(turn_idx, state)` | no | `True` | Rarely | -| `should_continue_after_turn(state)` | no | `True` | `AgenticMixin` returns False once assistant emits no tool calls | | `allow_early_stop_at_turn(turn_idx, state)` | no | `True` | `ToolCallingMixin` defers until tools are used; `safety_chat_pressure` returns False (the dedicated capitulation classifier on the assistant side is the only stop signal) | | `async is_capitulation_detected(state)` | no | `False` | `safety_chat_pressure` overrides this and runs a dedicated comparator-shaped classifier (`classifier.py::classify_capitulation`) against the assistant's last response. When True the simulator sets `stopped_on_capitulation=True`, `capitulated_at_turn=N`, emits `EARLY_STOP` + `CAPITULATION_CHECK` traces, and breaks the loop. Capitulation-specific by design: if a future probe needs a different post-assistant-turn stop reason, add a sibling hook with its own dedicated vocabulary rather than overloading this one. | | `user_turn_policy()` | no | `UserTurnPolicy()`: the loop's own behaviour | When the probe tests what the model says across turns: turn off compression so the model sees its own replies, turn off the wrap-up anchor so every follow-up stays on topic, allow filter phrases its users naturally write, and skip names (such as brand names in Latin script) in the script check. `identity_disclosure` does all four, and checks the openings its users write for AI names. A generated opening can also be checked before the gate (`check_opening`): return why a draft breaks the probe's rules, and the loop asks for another draft within `max_query_attempts`. Read once per trajectory, after `derive_task` | diff --git a/src/usersim/engine/core/probes.py b/src/usersim/engine/core/probes.py index 956f5e4..86211b6 100644 --- a/src/usersim/engine/core/probes.py +++ b/src/usersim/engine/core/probes.py @@ -145,7 +145,6 @@ class ProbeAdapter(Protocol): on_followup_failure(state, error_kind) -> str should_inline_judge_user_turn(turn_idx, state) -> bool should_inline_judge_assistant_turn(turn_idx, state) -> bool - should_continue_after_turn(state) -> bool allow_early_stop_at_turn(turn_idx, state) -> bool is_capitulation_detected(state) -> bool user_turn_policy() -> UserTurnPolicy @@ -516,15 +515,6 @@ def should_inline_judge_assistant_turn(self, turn_idx: int, state: Any) -> bool: """ return True - def should_continue_after_turn(self, state: Any) -> bool: - """Default True: continue to the next turn until ``cfg.max_turns``. - - ``AgenticMixin`` returns False as soon as the assistant - emits a turn with no tool calls (the agentic protocol is - single-decision: tool call OR refusal, no follow-up). - """ - return True - def allow_early_stop_at_turn(self, turn_idx: int, state: Any) -> bool: """Whether to consider early-stop heuristics at this turn. @@ -561,7 +551,7 @@ async def is_capitulation_detected(self, state: Any) -> bool: ``self._task`` / ``self._locale`` (set in ``BaseProbe.__init__``); no need to thread them through the hook signature. Matches the state-only convention of peer - hooks (``should_continue_after_turn(state)``, + hooks (``should_succeed(state)``, ``allow_early_stop_at_turn(turn_idx, state)``). Distinct from ``allow_early_stop_at_turn`` (which gates the @@ -973,8 +963,6 @@ class AgenticMixin(BankVerbatimMixin): Sets: - - ``should_continue_after_turn`` — False when the assistant's last - message has no tool calls (the protocol is single-decision). - ``should_inline_judge_user_turn`` — False for turn 0 (the verbatim action request is auditable as-is; gate burns judge calls for no signal on agentic probes). @@ -989,15 +977,6 @@ class AgenticMixin(BankVerbatimMixin): # is off (see the same attribute on ``ToolCallingMixin``). supports_assistant_resampling: bool = False - def should_continue_after_turn(self, state: Any) -> bool: - if not state.messages: - return False - last = state.messages[-1] - if last.get("role") != "assistant": - return False - tool_calls = last.get("tool_calls") - return bool(tool_calls) - def should_inline_judge_user_turn(self, turn_idx: int, state: Any) -> bool: return turn_idx > 0 diff --git a/templates/probe/generator_agentic.py b/templates/probe/generator_agentic.py index 739c4a1..ec976d5 100644 --- a/templates/probe/generator_agentic.py +++ b/templates/probe/generator_agentic.py @@ -22,8 +22,6 @@ What ``AgenticMixin`` does: - Inherits from ``BankVerbatimMixin``. -- Sets ``should_continue_after_turn = False`` once the assistant - emits a turn with no tool calls. - Sets ``should_inline_judge_user_turn = False`` for turn 0 (the user's verbatim action request is auditable as-is — no judge call). @@ -36,7 +34,8 @@ not "system prompt" — it's API surface). - ``run_dispatch(*, models, data, cfg)`` — your custom simulate loop. Inject verbatim user turn-1, loop assistant calls, intercept - tool calls + append mock responses. + tool calls + append mock responses, and stop when the assistant + answers without tool calls (as ``safety_agentic`` does). - ``build_result_extras`` — overrides ``num_turns`` to assistant- message count (NOT user-turn count — single user turn here). diff --git a/tests/engine/core/test_probes_substrate.py b/tests/engine/core/test_probes_substrate.py index beb244b..1b7e52a 100644 --- a/tests/engine/core/test_probes_substrate.py +++ b/tests/engine/core/test_probes_substrate.py @@ -159,7 +159,6 @@ async def test_optional_hooks_return_documented_defaults(self) -> None: assert probe.on_followup_failure(state, "gate_exhausted") == "skip" assert probe.should_inline_judge_user_turn(0, state) is True assert probe.should_inline_judge_assistant_turn(0, state) is True - assert probe.should_continue_after_turn(state) is True assert probe.allow_early_stop_at_turn(0, state) is True assert probe.user_turn_policy() == UserTurnPolicy() @@ -488,30 +487,6 @@ def _build(self) -> _AgenticProbe: provenance=Provenance(), ) - def test_continues_when_assistant_emitted_tool_calls(self) -> None: - probe = self._build() - state = _StubState( - messages=[ - {"role": "user"}, - {"role": "assistant", "tool_calls": [{"id": "t1"}]}, - ] - ) - assert probe.should_continue_after_turn(state) is True - - def test_stops_when_assistant_emitted_no_tool_calls(self) -> None: - probe = self._build() - state = _StubState( - messages=[ - {"role": "user"}, - {"role": "assistant", "content": "I refuse."}, - ] - ) - assert probe.should_continue_after_turn(state) is False - - def test_stops_on_empty_message_history(self) -> None: - probe = self._build() - assert probe.should_continue_after_turn(_StubState()) is False - def test_skips_user_judge_for_turn_zero(self) -> None: probe = self._build() state = _StubState()