From d7bffc3da1e0be85d63a4e5c0d3e4ffbbee56262 Mon Sep 17 00:00:00 2001 From: kelkalot Date: Sat, 24 Oct 2026 20:46:53 +0200 Subject: [PATCH 1/2] Pass targets a clean history, documents and system prompt - CallableTarget gives the function the history without the engine's markers, the scenario's documents as text through `documents`, and its files through `file_uri`, so no evaluation-only document mark reaches it (#126). - HTTPAppTarget sends the audit's system prompt: as a leading system message in "messages" mode, after which a `messages` list from request_template comes before the conversation; in other modes through the new `system_field`, with a warning when there is none (#127). - A `history` list in HTTPAppTarget's request_template comes before the conversation's earlier turns instead of replacing them (#128). - targets/base.py gains ENGINE_MARKERS, plain_history() and history_file_uri(), shared by both targets. Fixes #126 Fixes #127 Fixes #128 --- README.md | 8 ++++ simpleaudit/targets/base.py | 16 +++++++ simpleaudit/targets/callable.py | 26 ++++++++-- simpleaudit/targets/http.py | 50 +++++++++++++------ tests/test_targets.py | 85 +++++++++++++++++++++++++++++++++ 5 files changed, 166 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 4bf5032..ede9185 100644 --- a/README.md +++ b/README.md @@ -1084,6 +1084,14 @@ async def my_agent(*, user, history, context, **_): auditor = Auditor(target=CallableTarget(my_agent), judge_model="gpt-4o", judge_provider="openai") ``` +The function gets the conversation so far in `history`, ending with the current user turn, the scenario's documents as text in `documents`, and its attached files in `file_uri`. The documents' evaluation-only marks never reach it. + +For an app behind HTTP, `HTTPAppTarget(url=..., response_path=...)` sends each probe as a request body: + +- **`message_field="messages"`:** the body is an OpenAI-style message list. The audit's `system_prompt` is a leading system message, and any `messages` in `request_template` come before the conversation. +- **Other modes:** pass `system_field=` to send the system prompt. A `history` list in `request_template` comes before the conversation's earlier turns. +- **Documents** go to `documents_field`, as text. + A target class can also use these optional pieces: - **`start_scenario(context)` / `end_scenario(context)`**: async methods for targets that hold a session per scenario. `start_scenario` is awaited before the first turn. Once it has succeeded, `end_scenario` runs after the last turn and the evidence resolver, before judging, and also when a turn fails or the run is cancelled; a `start_scenario` that raises has to clean up after itself. `context.scenario_run_id` matches the per-turn contexts, and `context.extra["scenario_name"]` names the scenario. A hook defined without `async` is not called, and a warning says so. `SingleTurnAuditor` doesn't call them. diff --git a/simpleaudit/targets/base.py b/simpleaudit/targets/base.py index 1a66ed3..b237ce5 100644 --- a/simpleaudit/targets/base.py +++ b/simpleaudit/targets/base.py @@ -116,8 +116,24 @@ async def send( ... +#: The engine's own keys beside ``role`` and ``content`` on history turns (see +#: ``Target.send``). None of them is input for the system under test, and +#: documents carry evaluation-only marks. +ENGINE_MARKERS = ("documents", "file_uri", "decision", "tool_events") + + +def plain_history(history: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: + """The history without the engine's markers, as a new list; other fields stay.""" + return [{k: v for k, v in m.items() if k not in ENGINE_MARKERS} for m in history or []] + + def history_documents( history: Optional[List[Dict[str, Any]]], ) -> Optional[List[Union[str, Dict[str, Any]]]]: """A scenario's documents, which ``run_scenario`` carries on the history's first user turn.""" return next((m["documents"] for m in history or [] if m.get("documents")), None) + + +def history_file_uri(history: Optional[List[Dict[str, Any]]]) -> Optional[Union[str, List[str]]]: + """A scenario's attached files, which ``run_scenario`` carries on the history's first user turn.""" + return next((m["file_uri"] for m in history or [] if m.get("file_uri")), None) diff --git a/simpleaudit/targets/callable.py b/simpleaudit/targets/callable.py index fbf9c1b..5ae5cbc 100644 --- a/simpleaudit/targets/callable.py +++ b/simpleaudit/targets/callable.py @@ -14,14 +14,28 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Union -from .base import Target, TargetContext, TargetResponse +from ..context_marks import parse_documents +from .base import ( + Target, + TargetContext, + TargetResponse, + history_documents, + history_file_uri, + plain_history, +) # A callable target may return a string or a TargetResponse. TargetCallable = Callable[..., Union[str, TargetResponse, Awaitable[Union[str, TargetResponse]]]] class CallableTarget: - """Wrap an in-process callable as a :class:`Target`.""" + """Wrap an in-process callable as a :class:`Target`. + + The callable gets what a system under test may see (see ``Target.send``): + the history without the engine's markers, the scenario's documents as + text through ``documents`` and its files through ``file_uri``. No + evaluation-only document mark reaches it. + """ def __init__(self, fn: TargetCallable) -> None: self._fn = fn @@ -38,12 +52,14 @@ async def send( params: Optional[Dict[str, Any]] = None, context: Optional[TargetContext] = None, ) -> TargetResponse: + if documents is None: + documents = history_documents(history) result = self._fn( system=system, user=user, - history=history, - file_uri=file_uri, - documents=documents, + history=plain_history(history) if history is not None else None, + file_uri=file_uri if file_uri is not None else history_file_uri(history), + documents=[mark.text for mark in parse_documents(documents)] or None, response_format=response_format, params=params, context=context, diff --git a/simpleaudit/targets/http.py b/simpleaudit/targets/http.py index ad1f208..de12a98 100644 --- a/simpleaudit/targets/http.py +++ b/simpleaudit/targets/http.py @@ -20,10 +20,18 @@ from __future__ import annotations import json +import warnings from typing import Any, Callable, Dict, List, Optional, Union from ..context_marks import parse_documents -from .base import Target, TargetContext, TargetResponse, history_documents +from .base import ( + Target, + TargetContext, + TargetResponse, + history_documents, + history_file_uri, + plain_history, +) def _resolve_path(data: Any, path: Optional[str]) -> Any: @@ -44,11 +52,6 @@ def _resolve_path(data: Any, path: Optional[str]) -> Any: return cur -#: The audit engine's own keys beside ``role`` and ``content`` (see Target.send). -#: Documents carry evaluation-only marks, and none of them is the app's input. -_ENGINE_MARKERS = ("documents", "file_uri", "decision", "tool_events") - - def _earlier_turns(user: str, history: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: """The conversation before the current turn, without the engine's markers. @@ -56,7 +59,7 @@ def _earlier_turns(user: str, history: Optional[List[Dict[str, Any]]]) -> List[D the body adds itself; a direct caller may pass the earlier turns only. Fields other than the markers (``tool_calls``, ``tool_call_id``, ...) stay. """ - turns = [{k: v for k, v in m.items() if k not in _ENGINE_MARKERS} for m in history or []] + turns = plain_history(history) if turns and turns[-1].get("role") == "user" and turns[-1].get("content") == user: turns.pop() return turns @@ -84,6 +87,9 @@ def __init__( # Optional: extract token usage from the response, e.g. # ("usage.prompt_tokens", "usage.completion_tokens"). token_paths: Optional[tuple] = None, + # Where the audit's system prompt goes when message_field is not + # "messages" (in "messages" mode it is a leading system message). + system_field: Optional[str] = None, ) -> None: self.url = url self.method = method.upper() @@ -95,12 +101,15 @@ def __init__( self._client = client self.documents_field = documents_field self.token_paths = token_paths + self.system_field = system_field + self._warned_system = False def _build_body( self, user: str, history: Optional[List[Dict[str, Any]]], documents: Optional[List[Union[str, Dict[str, Any]]]] = None, + system: Optional[str] = None, ) -> Dict[str, Any]: body = json.loads(json.dumps(self.request_template)) # deep copy # Retrieval context: text only, so no evaluation-only mark ever @@ -123,14 +132,27 @@ def _build_body( body[self.documents_field] = doc_texts earlier = _earlier_turns(user, history) if self.message_field == "messages": - # OpenAI-style chat body: ``messages`` is a list of - # ``{"role", "content"}`` dicts. Append the current user turn, - # optionally preceded by prior conversation history. - body["messages"] = [*earlier, {"role": "user", "content": user}] + # OpenAI-style chat body: the audit's system prompt, any messages + # the request template fixes, then the conversation. + fixed = body.get("messages") if isinstance(body.get("messages"), list) else [] + system_turn = [{"role": "system", "content": system}] if system else [] + body["messages"] = [*system_turn, *fixed, *earlier, {"role": "user", "content": user}] else: body[self.message_field] = user + if system: + if self.system_field: + body[self.system_field] = system + elif not self._warned_system: + self._warned_system = True + warnings.warn( + "HTTPAppTarget has nowhere to send the system prompt: pass " + "system_field= to name the request field for it.", + stacklevel=3, + ) if history is not None: - body.setdefault("history", earlier) + # A history list in the template comes before the conversation. + fixed = body.get("history") if isinstance(body.get("history"), list) else [] + body["history"] = [*fixed, *earlier] return body async def send( @@ -147,10 +169,10 @@ async def send( ) -> TargetResponse: # Like documents, an attachment the app never receives must not be # graded as if it had. - if file_uri or any(m.get("file_uri") for m in history or []): + if file_uri or history_file_uri(history): raise ValueError("HTTPAppTarget cannot send attached files; this scenario attaches file_uri") headers = dict(self.headers) - body = self._build_body(user, history, documents) + body = self._build_body(user, history, documents, system) # Forward correlation / trace context when available. if context is not None: diff --git a/tests/test_targets.py b/tests/test_targets.py index ec76ecd..e4c1480 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -12,6 +12,8 @@ - Auditor(target=...) generic entry point """ +from pathlib import Path + import pytest from simpleaudit import ( @@ -523,3 +525,86 @@ def test_http_audit_with_attached_files_fails_instead_of_dropping_them(): result, bodies = _audit_http({**PLAIN_SCENARIO, "file_uri": "/tmp/scan.png"}, message_field="messages") assert result.severity == "ERROR" and "cannot send attached files" in result.judgment["error"] assert bodies == [] + + +# Issues #126–#128: what CallableTarget and HTTPAppTarget pass on. + + +def test_callable_gets_plain_history_and_text_documents(): + import asyncio + + from tests.fakes import fixed_probe_auditor, fixed_severity_judge, fixed_target, make_auditor + + seen = [] + auditor = make_auditor( + fixed_target("x"), fixed_severity_judge("pass"), fixed_probe_auditor("Second question?"), max_turns=2 + ) + auditor.set_target(CallableTarget(lambda **kwargs: seen.append(kwargs) or "ok")) + image = str(Path(__file__).resolve().parent.parent / "simpleaudit" / "scenarios" / "images" / "product_a.png") + scenario = {**PLAIN_SCENARIO, "documents": ["Appeals go to NAV.", MARKED_DOCUMENT], "file_uri": image} + result = asyncio.run(auditor.run_async([scenario]))[0] + assert result.severity == "pass" + first, second = seen + assert first["history"] == [{"role": "user", "content": "First question?"}] + assert second["history"] == [ + {"role": "user", "content": "First question?"}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "Second question?"}, + ] + assert first["documents"] == second["documents"] == ["Appeals go to NAV.", "Deadline is six weeks."] + assert first["file_uri"] == image + assert "file_uri" in result.conversation[0] # the stored transcript keeps its markers + + +@pytest.mark.asyncio +async def test_callable_passes_given_documents_and_no_history_through(): + seen = [] + await CallableTarget(lambda **kwargs: seen.append(kwargs) or "ok").send(user="q", documents=[MARKED_DOCUMENT]) + assert seen[0]["history"] is None and seen[0]["file_uri"] is None + assert seen[0]["documents"] == ["Deadline is six weeks."] + + +def test_http_messages_body_sends_the_system_prompt_and_fixed_messages(): + t = HTTPAppTarget( + url="http://x", + message_field="messages", + request_template={"model": "m", "messages": [{"role": "system", "content": "Fixed app prompt."}]}, + ) + history = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}, {"role": "user", "content": "more"}] + body = t._build_body("more", history, system="You are careful.") + assert body["messages"] == [ + {"role": "system", "content": "You are careful."}, + {"role": "system", "content": "Fixed app prompt."}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "more"}, + ] + assert t.request_template["messages"] == [{"role": "system", "content": "Fixed app prompt."}] + + +def test_http_message_body_sends_the_system_prompt_to_system_field_or_warns(): + import warnings + + assert HTTPAppTarget(url="http://x", system_field="instructions")._build_body("q", None, system="Be careful.") == { + "message": "q", + "instructions": "Be careful.", + } + t = HTTPAppTarget(url="http://x") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + t._build_body("q", None, system="Be careful.") + t._build_body("q", None, system="Be careful.") + assert [str(w.message) for w in caught if "system_field" in str(w.message)] != [] + assert len([w for w in caught if "system_field" in str(w.message)]) == 1 + + +def test_http_template_history_comes_before_the_conversation(): + _, bodies = _audit_http(PLAIN_SCENARIO, request_template={"history": []}) + assert bodies[1]["history"] == [ + {"role": "user", "content": "First question?"}, + {"role": "assistant", "content": "ok"}, + ] + fixed = [{"role": "user", "content": "Earlier session."}, {"role": "assistant", "content": "Noted."}] + _, bodies = _audit_http(PLAIN_SCENARIO, request_template={"history": fixed}) + assert bodies[0]["history"] == fixed + assert bodies[1]["history"] == [*fixed, {"role": "user", "content": "First question?"}, {"role": "assistant", "content": "ok"}] From a62f420e4ec0e0ad1be68812aa1e2334b107bb6c Mon Sep 17 00:00:00 2001 From: kelkalot Date: Sat, 24 Oct 2026 20:56:58 +0200 Subject: [PATCH 2/2] Refine system prompt routing and file passing after review - system_field is honoured in "messages" mode too, so APIs that take the system prompt as a top-level field (Anthropic's native Messages API) work. A value request_template already sets at that field comes first; content-block lists get the prompt appended as a text block. - Without system_field, the audit's prompt joins the template's leading system message instead of adding a second one; many chat templates reject two system messages in a row. - The missing-system_field warning fires on every call when warnings are errors, instead of failing once and then dropping the prompt silently. - A non-list "history" in request_template is an error instead of being replaced by the conversation. - CallableTarget passes file_uri on the turn that attaches it only, and that turn keeps its file_uri in the history, so a function can attach the files where they belong. - context_marks.document_texts() replaces the copies in the callable and HTTP targets; DecisionTarget treats an explicit documents=[] as given; OpenEnvTarget uses the shared history helpers. Stale comments and the README are updated. --- README.md | 7 ++-- simpleaudit/context_marks.py | 7 ++++ simpleaudit/single_turn.py | 8 ++-- simpleaudit/targets/base.py | 15 ++++++-- simpleaudit/targets/callable.py | 22 ++++++----- simpleaudit/targets/decision.py | 2 +- simpleaudit/targets/http.py | 65 +++++++++++++++++++++++---------- simpleaudit/targets/openenv.py | 4 +- tests/test_targets.py | 57 +++++++++++++++++++++++++---- 9 files changed, 136 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index ede9185..e1fe73a 100644 --- a/README.md +++ b/README.md @@ -1084,12 +1084,13 @@ async def my_agent(*, user, history, context, **_): auditor = Auditor(target=CallableTarget(my_agent), judge_model="gpt-4o", judge_provider="openai") ``` -The function gets the conversation so far in `history`, ending with the current user turn, the scenario's documents as text in `documents`, and its attached files in `file_uri`. The documents' evaluation-only marks never reach it. +In a multi-turn audit, the function gets the conversation so far in `history`, ending with the current user turn. It also gets the scenario's documents as text in `documents` on every turn, and its attached files in `file_uri` on the turn that attaches them. That turn keeps its `file_uri` in `history`. The documents' evaluation-only marks never reach the function. For an app behind HTTP, `HTTPAppTarget(url=..., response_path=...)` sends each probe as a request body: -- **`message_field="messages"`:** the body is an OpenAI-style message list. The audit's `system_prompt` is a leading system message, and any `messages` in `request_template` come before the conversation. -- **Other modes:** pass `system_field=` to send the system prompt. A `history` list in `request_template` comes before the conversation's earlier turns. +- **`message_field="messages"`:** the body is an OpenAI-style message list, and any `messages` in `request_template` come before the conversation. The audit's `system_prompt` joins the template's leading system message, or becomes one. +- **`system_field=`:** names a request field for the system prompt instead, in either mode. A value the template already sets there comes first. +- **Other modes:** a `history` list in `request_template` comes before the conversation's earlier turns. - **Documents** go to `documents_field`, as text. A target class can also use these optional pieces: diff --git a/simpleaudit/context_marks.py b/simpleaudit/context_marks.py index 688912d..93a1f1b 100644 --- a/simpleaudit/context_marks.py +++ b/simpleaudit/context_marks.py @@ -258,6 +258,13 @@ def _as_marks( return parse_documents(marks) +def document_texts( + docs: Optional[Sequence[Union[str, Dict[str, Any], DocumentMark]]], +) -> List[str]: + """The documents' text only, in order: what a target may receive.""" + return [mark.text for mark in parse_documents(docs)] + + def render_documents( marks: Sequence[Union[DocumentMark, str, Dict[str, Any]]], ) -> str: diff --git a/simpleaudit/single_turn.py b/simpleaudit/single_turn.py index ca76282..4aa393d 100644 --- a/simpleaudit/single_turn.py +++ b/simpleaudit/single_turn.py @@ -318,11 +318,9 @@ async def _run_one_scenario( marks = parse_documents(documents) # Targets must see document TEXT only. The marks (relevant / true / # decisive / valid_until / authority) are evaluation-only ground truth; - # strip them here, at the single choke point, so no target — callable, - # HTTP, model, or decision — can read them. This is the one place the - # raw marked dicts could otherwise reach a target (the model/decision - # targets already strip to .text themselves, but the callable and HTTP - # targets forward `documents` verbatim). + # strip them here, so no target can read them. The built-in targets + # also reduce documents to text themselves, but a custom target may + # forward `documents` as it gets them. target_documents = [mark.text for mark in marks] if marks else None as_of = parse_as_of(scenario) derivations = derive_all(marks, as_of) diff --git a/simpleaudit/targets/base.py b/simpleaudit/targets/base.py index b237ce5..b0b918e 100644 --- a/simpleaudit/targets/base.py +++ b/simpleaudit/targets/base.py @@ -10,7 +10,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Protocol, Union, runtime_checkable +from typing import Any, Dict, List, Optional, Protocol, Sequence, Union, runtime_checkable @dataclass @@ -122,9 +122,16 @@ async def send( ENGINE_MARKERS = ("documents", "file_uri", "decision", "tool_events") -def plain_history(history: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: - """The history without the engine's markers, as a new list; other fields stay.""" - return [{k: v for k, v in m.items() if k not in ENGINE_MARKERS} for m in history or []] +def plain_history( + history: Optional[List[Dict[str, Any]]], + keep: Sequence[str] = (), +) -> List[Dict[str, Any]]: + """The history without the engine's markers (bar those in ``keep``), as a new list. + + Fields other than the markers stay. + """ + drop = [k for k in ENGINE_MARKERS if k not in keep] + return [{k: v for k, v in m.items() if k not in drop} for m in history or []] def history_documents( diff --git a/simpleaudit/targets/callable.py b/simpleaudit/targets/callable.py index 5ae5cbc..c959e4f 100644 --- a/simpleaudit/targets/callable.py +++ b/simpleaudit/targets/callable.py @@ -6,21 +6,21 @@ - auditing a local function that wraps an agent, RAG pipeline, or tool - quick experiments before wiring up a real HTTP endpoint -The callable receives the same logical inputs as any Target and must return -either a plain string (treated as ``content``) or a :class:`TargetResponse`. +The callable receives the inputs a system under test may see (see +``CallableTarget``) and must return either a plain string (treated as +``content``) or a :class:`TargetResponse`. """ from __future__ import annotations from typing import Any, Awaitable, Callable, Dict, List, Optional, Union -from ..context_marks import parse_documents +from ..context_marks import document_texts from .base import ( Target, TargetContext, TargetResponse, history_documents, - history_file_uri, plain_history, ) @@ -33,8 +33,10 @@ class CallableTarget: The callable gets what a system under test may see (see ``Target.send``): the history without the engine's markers, the scenario's documents as - text through ``documents`` and its files through ``file_uri``. No - evaluation-only document mark reaches it. + text through ``documents`` on every turn, and its files through + ``file_uri`` on the turn that attaches them. That turn keeps its + ``file_uri`` in the history, so a function can attach the files where + they belong. No evaluation-only document mark reaches it. """ def __init__(self, fn: TargetCallable) -> None: @@ -54,12 +56,14 @@ async def send( ) -> TargetResponse: if documents is None: documents = history_documents(history) + if file_uri is None and history and history[-1].get("file_uri"): + file_uri = history[-1]["file_uri"] result = self._fn( system=system, user=user, - history=plain_history(history) if history is not None else None, - file_uri=file_uri if file_uri is not None else history_file_uri(history), - documents=[mark.text for mark in parse_documents(documents)] or None, + history=plain_history(history, keep=("file_uri",)) if history is not None else None, + file_uri=file_uri, + documents=document_texts(documents) or None, response_format=response_format, params=params, context=context, diff --git a/simpleaudit/targets/decision.py b/simpleaudit/targets/decision.py index 90fdc4a..7fcab6f 100644 --- a/simpleaudit/targets/decision.py +++ b/simpleaudit/targets/decision.py @@ -159,7 +159,7 @@ async def send( "(see the scenario guidelines, 'Decision Field')" ) # The auditor attaches a scenario's documents to the first user turn. - if not documents: + if documents is None: documents = history_documents(history) body = self.build_request(decision, user, documents) payload = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8") diff --git a/simpleaudit/targets/http.py b/simpleaudit/targets/http.py index de12a98..0e86df1 100644 --- a/simpleaudit/targets/http.py +++ b/simpleaudit/targets/http.py @@ -23,7 +23,7 @@ import warnings from typing import Any, Callable, Dict, List, Optional, Union -from ..context_marks import parse_documents +from ..context_marks import document_texts from .base import ( Target, TargetContext, @@ -65,6 +65,26 @@ def _earlier_turns(user: str, history: Optional[List[Dict[str, Any]]]) -> List[D return turns +def _with_system_text(fixed: Any, system: str) -> Any: + """The template's own system prompt at that field, if any, followed by the audit's.""" + if isinstance(fixed, str) and fixed: + return f"{fixed}\n\n{system}" + if isinstance(fixed, list): # content blocks, e.g. Anthropic's native "system" + return [*fixed, {"type": "text", "text": system}] + return system + + +def _with_system_message(messages: List[Dict[str, Any]], system: str) -> List[Dict[str, Any]]: + """One leading system message: the template's, if it starts with one, then the audit's prompt. + + Many chat templates reject two system messages in a row. + """ + if messages and messages[0].get("role") == "system": + first = messages[0] + return [{**first, "content": _with_system_text(first.get("content"), system)}, *messages[1:]] + return [{"role": "system", "content": system}, *messages] + + class HTTPAppTarget: """Send audit probes to an external HTTP application and parse the answer.""" @@ -87,8 +107,9 @@ def __init__( # Optional: extract token usage from the response, e.g. # ("usage.prompt_tokens", "usage.completion_tokens"). token_paths: Optional[tuple] = None, - # Where the audit's system prompt goes when message_field is not - # "messages" (in "messages" mode it is a leading system message). + # The request field for the audit's system prompt. Without it, the + # prompt is a leading system message in "messages" mode and cannot be + # sent in other modes. system_field: Optional[str] = None, ) -> None: self.url = url @@ -121,7 +142,7 @@ def _build_body( # the context for. if documents is None: documents = history_documents(history) - doc_texts = [mark.text for mark in parse_documents(documents)] + doc_texts = document_texts(documents) if doc_texts: if self.documents_field is None: raise ValueError( @@ -131,27 +152,33 @@ def _build_body( ) body[self.documents_field] = doc_texts earlier = _earlier_turns(user, history) + if system and self.system_field: + body[self.system_field] = _with_system_text(body.get(self.system_field), system) if self.message_field == "messages": - # OpenAI-style chat body: the audit's system prompt, any messages - # the request template fixes, then the conversation. + # OpenAI-style chat body: any messages the request template fixes, + # then the conversation. Without a system_field, the audit's system + # prompt joins the template's leading system message or becomes one. fixed = body.get("messages") if isinstance(body.get("messages"), list) else [] - system_turn = [{"role": "system", "content": system}] if system else [] - body["messages"] = [*system_turn, *fixed, *earlier, {"role": "user", "content": user}] + if system and not self.system_field: + fixed = _with_system_message(fixed, system) + body["messages"] = [*fixed, *earlier, {"role": "user", "content": user}] else: body[self.message_field] = user - if system: - if self.system_field: - body[self.system_field] = system - elif not self._warned_system: - self._warned_system = True - warnings.warn( - "HTTPAppTarget has nowhere to send the system prompt: pass " - "system_field= to name the request field for it.", - stacklevel=3, - ) + if system and not self.system_field and not self._warned_system: + warnings.warn( + "HTTPAppTarget has nowhere to send the system prompt: pass " + "system_field= to name the request field for it.", + stacklevel=3, + ) + self._warned_system = True if history is not None: # A history list in the template comes before the conversation. - fixed = body.get("history") if isinstance(body.get("history"), list) else [] + fixed = body.get("history", []) + if not isinstance(fixed, list): + raise ValueError( + "request_template's 'history' must be a list for the conversation to " + "follow it." + ) body["history"] = [*fixed, *earlier] return body diff --git a/simpleaudit/targets/openenv.py b/simpleaudit/targets/openenv.py index d838ea5..0937a54 100644 --- a/simpleaudit/targets/openenv.py +++ b/simpleaudit/targets/openenv.py @@ -26,7 +26,7 @@ from dataclasses import dataclass, field from typing import Any -from .base import TargetContext, TargetResponse +from .base import TargetContext, TargetResponse, history_documents, history_file_uri ResetKwargs = Mapping[str, Any] | Callable[[str | None], Mapping[str, Any]] @@ -171,7 +171,7 @@ async def send( ) # The engine carries a scenario's attachments as markers on the # history's first user turn, not as send() arguments. - if file_uri or documents or any(m.get("file_uri") or m.get("documents") for m in history or []): + if file_uri or documents or history_file_uri(history) or history_documents(history): raise ValueError("OpenEnvTarget sends text only; this scenario attaches files or documents") if session.done: raise RuntimeError("The agent already ended this episode") diff --git a/tests/test_targets.py b/tests/test_targets.py index e4c1480..013719a 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -27,7 +27,6 @@ ) from simpleaudit.targets.http import _resolve_path - # --------------------------------------------------------------------------- # TargetResponse / TargetContext # --------------------------------------------------------------------------- @@ -545,15 +544,16 @@ def test_callable_gets_plain_history_and_text_documents(): result = asyncio.run(auditor.run_async([scenario]))[0] assert result.severity == "pass" first, second = seen - assert first["history"] == [{"role": "user", "content": "First question?"}] + # The file stays on the turn that attached it, and file_uri is set on that turn only. + assert first["history"] == [{"role": "user", "content": "First question?", "file_uri": image}] assert second["history"] == [ - {"role": "user", "content": "First question?"}, + {"role": "user", "content": "First question?", "file_uri": image}, {"role": "assistant", "content": "ok"}, {"role": "user", "content": "Second question?"}, ] + assert (first["file_uri"], second["file_uri"]) == (image, None) assert first["documents"] == second["documents"] == ["Appeals go to NAV.", "Deadline is six weeks."] - assert first["file_uri"] == image - assert "file_uri" in result.conversation[0] # the stored transcript keeps its markers + assert "documents" in result.conversation[0] # the stored transcript keeps its markers @pytest.mark.asyncio @@ -573,8 +573,7 @@ def test_http_messages_body_sends_the_system_prompt_and_fixed_messages(): history = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}, {"role": "user", "content": "more"}] body = t._build_body("more", history, system="You are careful.") assert body["messages"] == [ - {"role": "system", "content": "You are careful."}, - {"role": "system", "content": "Fixed app prompt."}, + {"role": "system", "content": "Fixed app prompt.\n\nYou are careful."}, {"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}, {"role": "user", "content": "more"}, @@ -594,8 +593,14 @@ def test_http_message_body_sends_the_system_prompt_to_system_field_or_warns(): warnings.simplefilter("always") t._build_body("q", None, system="Be careful.") t._build_body("q", None, system="Be careful.") - assert [str(w.message) for w in caught if "system_field" in str(w.message)] != [] assert len([w for w in caught if "system_field" in str(w.message)]) == 1 + # With warnings as errors, every call fails, not just the first. + strict = HTTPAppTarget(url="http://x") + with warnings.catch_warnings(): + warnings.simplefilter("error") + for _ in range(2): + with pytest.raises(UserWarning): + strict._build_body("q", None, system="Be careful.") def test_http_template_history_comes_before_the_conversation(): @@ -608,3 +613,39 @@ def test_http_template_history_comes_before_the_conversation(): _, bodies = _audit_http(PLAIN_SCENARIO, request_template={"history": fixed}) assert bodies[0]["history"] == fixed assert bodies[1]["history"] == [*fixed, {"role": "user", "content": "First question?"}, {"role": "assistant", "content": "ok"}] + + + +def test_http_system_field_wins_in_messages_mode_and_joins_the_template_value(): + t = HTTPAppTarget(url="http://x", message_field="messages", system_field="system") + assert t._build_body("q", None, system="Be careful.") == { + "system": "Be careful.", + "messages": [{"role": "user", "content": "q"}], + } + joined = HTTPAppTarget(url="http://x", system_field="instructions", request_template={"instructions": "Fixed."}) + assert joined._build_body("q", None, system="Be careful.")["instructions"] == "Fixed.\n\nBe careful." + blocks = HTTPAppTarget( + url="http://x", message_field="messages", system_field="system", + request_template={"system": [{"type": "text", "text": "Fixed."}]}, + ) + assert blocks._build_body("q", None, system="Be careful.")["system"] == [ + {"type": "text", "text": "Fixed."}, + {"type": "text", "text": "Be careful."}, + ] + + +def test_http_template_messages_without_a_system_prompt_are_kept_as_they_are(): + fixed = [{"role": "user", "content": "Example question"}, {"role": "assistant", "content": "Example answer"}] + t = HTTPAppTarget(url="http://x", message_field="messages", request_template={"messages": fixed}) + assert t._build_body("q", None)["messages"] == [*fixed, {"role": "user", "content": "q"}] + assert t._build_body("q", None, system="Be careful.")["messages"] == [ + {"role": "system", "content": "Be careful."}, + *fixed, + {"role": "user", "content": "q"}, + ] + + +def test_http_template_history_that_is_not_a_list_is_an_error(): + t = HTTPAppTarget(url="http://x", request_template={"history": {"session": "abc"}}) + with pytest.raises(ValueError, match="must be a list"): + t._build_body("q", [{"role": "user", "content": "q"}])