fix(engine): ask the gate judge for the rating the parser reads

docs/engine/AUTHORING_A_PROBE.md

@@ -469,8 +469,8 @@ failure mode.
 
 The gate prompt asks an LLM judge whether each user turn is a
 legitimate user message vs a role-violation / fourth-wall break /
-off-topic content. The judge's PASS/FAIL determines whether the turn
-is accepted or the loop retries.
+off-topic content. The judge's rating decides whether the turn is
+accepted or the loop generates it again.
 
 `BaseProbe.format_gate_prompt` ships a generic role-violation rubric.
 Override when the gate should be probe-grounded: every inclusion /
@@ -488,10 +488,22 @@ def format_gate_prompt(self, user_query: str, conversation_history: str) -> str:
     )
 ```
 
-The judge response must include `<rating>PASS</rating>` or
-`<rating>FAIL</rating>`: `run_inline_judge` parses XML, not plain
-text. If your prompt doesn't request that format, the judge will
-return prose and the gate will fail-retry until exhausted.
+The prompt must ask for the reply `run_inline_judge` parses: an
+`<explanation>` tag, and a `<rating>` tag holding `success`, `warn`
+or `failure`. `success` and `warn` accept the turn. A reply without a
+valid rating gets one follow-up asking the judge to reformat it, and
+if that reply has none either, the turn counts as failed. End the
+prompt with the format the default rubric asks for:
+
+```text
+Strictly respond in the following format:
+<explanation>
+[A brief explanation of the rating.]
+</explanation>
+<rating>
+[Must be either 'success' or 'failure' strictly]
+</rating>
+```
 
 ---
 

src/usersim/engine/core/probes.py

@@ -212,17 +212,23 @@ def build_result_extras(self, state: Any) -> dict:
 
 
 # Default user-judge prompt template for probes that do not override
-# ``format_gate_prompt``. Ships behind a sentinel so importers cannot
-# accidentally hard-code "what the default rubric used to say"; if a
-# probe needs the default, it overrides explicitly.
+# ``format_gate_prompt``. It asks for the ``<explanation>`` and
+# ``<rating>`` reply that ``run_inline_judge`` parses.
 _DEFAULT_GATE_PROMPT_TEMPLATE = (
     "You are evaluating whether a user message in a conversation "
     "is a legitimate user turn (vs. role-violation, fourth-wall "
     "break, or off-topic content).\n\n"
     "Conversation so far:\n{conversation_history}\n\n"
     "User message to evaluate:\n{user_turn_to_evaluate}\n\n"
-    "Respond with PASS or FAIL on the first line, then a brief "
-    "explanation."
+    "Rate a legitimate user turn success, and a role violation, "
+    "fourth-wall break or off-topic content failure.\n\n"
+    "Strictly respond in the following format:\n"
+    "<explanation>\n"
+    "[A brief explanation of the rating.]\n"
+    "</explanation>\n"
+    "<rating>\n"
+    "[Must be either 'success' or 'failure' strictly]\n"
+    "</rating>"
 )
 
 

templates/probe/prompts.py

@@ -60,10 +60,15 @@
 
 DEMO_GATE_PROMPT_TEMPLATE = (
     "You are a strict judge. Decide whether the user message below is "
-    "a legitimate user turn (PASS) or a role-violation / fourth-wall "
-    "break / off-topic content (FAIL).\n\n"
+    "a legitimate user turn (success) or a role-violation / fourth-wall "
+    "break / off-topic content (failure).\n\n"
     "Conversation so far:\n{history}\n\n"
     "User message to evaluate:\n{user_turn}\n\n"
-    "Respond with PASS or FAIL on the first line, then a brief "
-    "explanation."
+    "Strictly respond in the following format:\n"
+    "<explanation>\n"
+    "[A brief explanation of the rating.]\n"
+    "</explanation>\n"
+    "<rating>\n"
+    "[Must be either 'success' or 'failure' strictly]\n"
+    "</rating>"
 )

tests/engine/test_hosted_parity_contract.py

@@ -6,7 +6,8 @@
 ``run_episode`` runs a probe offline with scripted models, and every registered
 probe reproduces that standalone run when hosted. Both go through the same
 ``usersim.testing`` entry points a probe author would use, so the published
-checks are exercised by the probes they describe.
+checks are exercised by the probes they describe. The requests a probe sends
+to the judge ask for the reply the judge parser reads.
 """
 
 from __future__ import annotations
@@ -84,6 +85,28 @@ async def test_every_registered_probe_runs_to_completion_with_scripted_models(pr
     assert run.prompts and all(prompt["messages"] for prompt in run.prompts)
 
 
+#: Probes with no simulated user, which run their own loop and send nothing to the judge.
+_NO_JUDGE_REQUESTS = frozenset({"safety_agentic"})
+
+
+@pytest.mark.parametrize("probe_type", known_probes())
+async def test_every_judge_request_asks_for_a_rating_the_parser_reads(probe_type: str) -> None:
+    """``run_inline_judge`` reads an ``<explanation>`` and a ``<rating>`` of success, warn or failure.
+
+    A prompt that asks for any other reply costs a reformat call on every turn it judges.
+    """
+    run = await run_episode(
+        probe_type, config=_config(probe_type), persona=_PERSONA, data=_data(probe_type), profile=_PROFILE
+    )
+
+    asked = [prompt["messages"][-1]["content"] for prompt in run.prompts if prompt["role"] == "judge"]
+    assert bool(asked) is (probe_type not in _NO_JUDGE_REQUESTS)
+    for text in asked:
+        assert "<explanation>" in text and "<rating>" in text, text
+        rating_format = text.rsplit("<rating>", 1)[1]
+        assert "success" in rating_format and "failure" in rating_format, text
+
+
 @pytest.mark.parametrize(("probe_type", "shares_details"), [("tool_calling", True), ("general_open_ended", False)])
 async def test_only_the_tool_calling_user_may_share_details(probe_type: str, shares_details: bool) -> None:
     """Its user hands the assistant the identifiers a tool call needs; an asking user gives only what is asked."""