diff --git a/docs/engine/AUTHORING_A_PROBE.md b/docs/engine/AUTHORING_A_PROBE.md index 6d1b81b..b26ccff 100644 --- a/docs/engine/AUTHORING_A_PROBE.md +++ b/docs/engine/AUTHORING_A_PROBE.md @@ -469,8 +469,8 @@ failure mode. The gate prompt asks an LLM judge whether each user turn is a legitimate user message vs a role-violation / fourth-wall break / -off-topic content. The judge's PASS/FAIL determines whether the turn -is accepted or the loop retries. +off-topic content. The judge's rating decides whether the turn is +accepted or the loop generates it again. `BaseProbe.format_gate_prompt` ships a generic role-violation rubric. Override when the gate should be probe-grounded: every inclusion / @@ -488,10 +488,22 @@ def format_gate_prompt(self, user_query: str, conversation_history: str) -> str: ) ``` -The judge response must include `PASS` or -`FAIL`: `run_inline_judge` parses XML, not plain -text. If your prompt doesn't request that format, the judge will -return prose and the gate will fail-retry until exhausted. +The prompt must ask for the reply `run_inline_judge` parses: an +`` tag, and a `` tag holding `success`, `warn` +or `failure`. `success` and `warn` accept the turn. A reply without a +valid rating gets one follow-up asking the judge to reformat it, and +if that reply has none either, the turn counts as failed. End the +prompt with the format the default rubric asks for: + +```text +Strictly respond in the following format: + +[A brief explanation of the rating.] + + +[Must be either 'success' or 'failure' strictly] + +``` --- diff --git a/src/usersim/engine/core/probes.py b/src/usersim/engine/core/probes.py index 835b0a9..ecf067c 100644 --- a/src/usersim/engine/core/probes.py +++ b/src/usersim/engine/core/probes.py @@ -212,17 +212,23 @@ def build_result_extras(self, state: Any) -> dict: # Default user-judge prompt template for probes that do not override -# ``format_gate_prompt``. Ships behind a sentinel so importers cannot -# accidentally hard-code "what the default rubric used to say"; if a -# probe needs the default, it overrides explicitly. +# ``format_gate_prompt``. It asks for the ```` and +# ```` reply that ``run_inline_judge`` parses. _DEFAULT_GATE_PROMPT_TEMPLATE = ( "You are evaluating whether a user message in a conversation " "is a legitimate user turn (vs. role-violation, fourth-wall " "break, or off-topic content).\n\n" "Conversation so far:\n{conversation_history}\n\n" "User message to evaluate:\n{user_turn_to_evaluate}\n\n" - "Respond with PASS or FAIL on the first line, then a brief " - "explanation." + "Rate a legitimate user turn success, and a role violation, " + "fourth-wall break or off-topic content failure.\n\n" + "Strictly respond in the following format:\n" + "\n" + "[A brief explanation of the rating.]\n" + "\n" + "\n" + "[Must be either 'success' or 'failure' strictly]\n" + "" ) diff --git a/templates/probe/prompts.py b/templates/probe/prompts.py index dce3c6e..67d67e2 100644 --- a/templates/probe/prompts.py +++ b/templates/probe/prompts.py @@ -60,10 +60,15 @@ DEMO_GATE_PROMPT_TEMPLATE = ( "You are a strict judge. Decide whether the user message below is " - "a legitimate user turn (PASS) or a role-violation / fourth-wall " - "break / off-topic content (FAIL).\n\n" + "a legitimate user turn (success) or a role-violation / fourth-wall " + "break / off-topic content (failure).\n\n" "Conversation so far:\n{history}\n\n" "User message to evaluate:\n{user_turn}\n\n" - "Respond with PASS or FAIL on the first line, then a brief " - "explanation." + "Strictly respond in the following format:\n" + "\n" + "[A brief explanation of the rating.]\n" + "\n" + "\n" + "[Must be either 'success' or 'failure' strictly]\n" + "" ) diff --git a/tests/engine/test_hosted_parity_contract.py b/tests/engine/test_hosted_parity_contract.py index 2119a25..2e60722 100644 --- a/tests/engine/test_hosted_parity_contract.py +++ b/tests/engine/test_hosted_parity_contract.py @@ -6,7 +6,8 @@ ``run_episode`` runs a probe offline with scripted models, and every registered probe reproduces that standalone run when hosted. Both go through the same ``usersim.testing`` entry points a probe author would use, so the published -checks are exercised by the probes they describe. +checks are exercised by the probes they describe. The requests a probe sends +to the judge ask for the reply the judge parser reads. """ from __future__ import annotations @@ -84,6 +85,28 @@ async def test_every_registered_probe_runs_to_completion_with_scripted_models(pr assert run.prompts and all(prompt["messages"] for prompt in run.prompts) +#: Probes with no simulated user, which run their own loop and send nothing to the judge. +_NO_JUDGE_REQUESTS = frozenset({"safety_agentic"}) + + +@pytest.mark.parametrize("probe_type", known_probes()) +async def test_every_judge_request_asks_for_a_rating_the_parser_reads(probe_type: str) -> None: + """``run_inline_judge`` reads an ```` and a ```` of success, warn or failure. + + A prompt that asks for any other reply costs a reformat call on every turn it judges. + """ + run = await run_episode( + probe_type, config=_config(probe_type), persona=_PERSONA, data=_data(probe_type), profile=_PROFILE + ) + + asked = [prompt["messages"][-1]["content"] for prompt in run.prompts if prompt["role"] == "judge"] + assert bool(asked) is (probe_type not in _NO_JUDGE_REQUESTS) + for text in asked: + assert "" in text and "" in text, text + rating_format = text.rsplit("", 1)[1] + assert "success" in rating_format and "failure" in rating_format, text + + @pytest.mark.parametrize(("probe_type", "shares_details"), [("tool_calling", True), ("general_open_ended", False)]) async def test_only_the_tool_calling_user_may_share_details(probe_type: str, shares_details: bool) -> None: """Its user hands the assistant the identifiers a tool call needs; an asking user gives only what is asked."""