Targets: clean history and documents for CallableTarget, system prompt and template history for HTTPAppTarget

README.md

@@ -1084,6 +1084,15 @@ async def my_agent(*, user, history, context, **_):
 auditor = Auditor(target=CallableTarget(my_agent), judge_model="gpt-4o", judge_provider="openai")
 ```
 
+In a multi-turn audit, the function gets the conversation so far in `history`, ending with the current user turn. It also gets the scenario's documents as text in `documents` on every turn, and its attached files in `file_uri` on the turn that attaches them. That turn keeps its `file_uri` in `history`. The documents' evaluation-only marks never reach the function.
+
+For an app behind HTTP, `HTTPAppTarget(url=..., response_path=...)` sends each probe as a request body:
+
+- **`message_field="messages"`:** the body is an OpenAI-style message list, and any `messages` in `request_template` come before the conversation. The audit's `system_prompt` joins the template's leading system message, or becomes one.
+- **`system_field=`:** names a request field for the system prompt instead, in either mode. A value the template already sets there comes first.
+- **Other modes:** a `history` list in `request_template` comes before the conversation's earlier turns.
+- **Documents** go to `documents_field`, as text.
+
 A target class can also use these optional pieces:
 
 - **`start_scenario(context)` / `end_scenario(context)`**: async methods for targets that hold a session per scenario. `start_scenario` is awaited before the first turn. Once it has succeeded, `end_scenario` runs after the last turn and the evidence resolver, before judging, and also when a turn fails or the run is cancelled; a `start_scenario` that raises has to clean up after itself. `context.scenario_run_id` matches the per-turn contexts, and `context.extra["scenario_name"]` names the scenario. A hook defined without `async` is not called, and a warning says so. `SingleTurnAuditor` doesn't call them.

simpleaudit/context_marks.py

@@ -258,6 +258,13 @@ def _as_marks(
     return parse_documents(marks)
 
 
+def document_texts(
+    docs: Optional[Sequence[Union[str, Dict[str, Any], DocumentMark]]],
+) -> List[str]:
+    """The documents' text only, in order: what a target may receive."""
+    return [mark.text for mark in parse_documents(docs)]
+
+
 def render_documents(
     marks: Sequence[Union[DocumentMark, str, Dict[str, Any]]],
 ) -> str:

simpleaudit/single_turn.py

@@ -318,11 +318,9 @@ async def _run_one_scenario(
         marks = parse_documents(documents)
         # Targets must see document TEXT only. The marks (relevant / true /
         # decisive / valid_until / authority) are evaluation-only ground truth;
-        # strip them here, at the single choke point, so no target — callable,
-        # HTTP, model, or decision — can read them. This is the one place the
-        # raw marked dicts could otherwise reach a target (the model/decision
-        # targets already strip to .text themselves, but the callable and HTTP
-        # targets forward `documents` verbatim).
+        # strip them here, so no target can read them. The built-in targets
+        # also reduce documents to text themselves, but a custom target may
+        # forward `documents` as it gets them.
         target_documents = [mark.text for mark in marks] if marks else None
         as_of = parse_as_of(scenario)
         derivations = derive_all(marks, as_of)

simpleaudit/targets/base.py

@@ -10,7 +10,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Protocol, Union, runtime_checkable
+from typing import Any, Dict, List, Optional, Protocol, Sequence, Union, runtime_checkable
 
 
 @dataclass
@@ -116,8 +116,31 @@ async def send(
         ...
 
 
+#: The engine's own keys beside ``role`` and ``content`` on history turns (see
+#: ``Target.send``). None of them is input for the system under test, and
+#: documents carry evaluation-only marks.
+ENGINE_MARKERS = ("documents", "file_uri", "decision", "tool_events")
+
+
+def plain_history(
+    history: Optional[List[Dict[str, Any]]],
+    keep: Sequence[str] = (),
+) -> List[Dict[str, Any]]:
+    """The history without the engine's markers (bar those in ``keep``), as a new list.
+
+    Fields other than the markers stay.
+    """
+    drop = [k for k in ENGINE_MARKERS if k not in keep]
+    return [{k: v for k, v in m.items() if k not in drop} for m in history or []]
+
+
 def history_documents(
     history: Optional[List[Dict[str, Any]]],
 ) -> Optional[List[Union[str, Dict[str, Any]]]]:
     """A scenario's documents, which ``run_scenario`` carries on the history's first user turn."""
     return next((m["documents"] for m in history or [] if m.get("documents")), None)
+
+
+def history_file_uri(history: Optional[List[Dict[str, Any]]]) -> Optional[Union[str, List[str]]]:
+    """A scenario's attached files, which ``run_scenario`` carries on the history's first user turn."""
+    return next((m["file_uri"] for m in history or [] if m.get("file_uri")), None)

simpleaudit/targets/callable.py

@@ -6,22 +6,38 @@
     - auditing a local function that wraps an agent, RAG pipeline, or tool
     - quick experiments before wiring up a real HTTP endpoint
 
-The callable receives the same logical inputs as any Target and must return
-either a plain string (treated as ``content``) or a :class:`TargetResponse`.
+The callable receives the inputs a system under test may see (see
+``CallableTarget``) and must return either a plain string (treated as
+``content``) or a :class:`TargetResponse`.
 """
 
 from __future__ import annotations
 
 from typing import Any, Awaitable, Callable, Dict, List, Optional, Union
 
-from .base import Target, TargetContext, TargetResponse
+from ..context_marks import document_texts
+from .base import (
+    Target,
+    TargetContext,
+    TargetResponse,
+    history_documents,
+    plain_history,
+)
 
 # A callable target may return a string or a TargetResponse.
 TargetCallable = Callable[..., Union[str, TargetResponse, Awaitable[Union[str, TargetResponse]]]]
 
 
 class CallableTarget:
-    """Wrap an in-process callable as a :class:`Target`."""
+    """Wrap an in-process callable as a :class:`Target`.
+
+    The callable gets what a system under test may see (see ``Target.send``):
+    the history without the engine's markers, the scenario's documents as
+    text through ``documents`` on every turn, and its files through
+    ``file_uri`` on the turn that attaches them. That turn keeps its
+    ``file_uri`` in the history, so a function can attach the files where
+    they belong. No evaluation-only document mark reaches it.
+    """
 
     def __init__(self, fn: TargetCallable) -> None:
         self._fn = fn
@@ -38,12 +54,16 @@ async def send(
         params: Optional[Dict[str, Any]] = None,
         context: Optional[TargetContext] = None,
     ) -> TargetResponse:
+        if documents is None:
+            documents = history_documents(history)
+        if file_uri is None and history and history[-1].get("file_uri"):
+            file_uri = history[-1]["file_uri"]
         result = self._fn(
             system=system,
             user=user,
-            history=history,
+            history=plain_history(history, keep=("file_uri",)) if history is not None else None,
             file_uri=file_uri,
-            documents=documents,
+            documents=document_texts(documents) or None,
             response_format=response_format,
             params=params,
             context=context,

simpleaudit/targets/decision.py

@@ -159,7 +159,7 @@ async def send(
                 "(see the scenario guidelines, 'Decision Field')"
             )
         # The auditor attaches a scenario's documents to the first user turn.
-        if not documents:
+        if documents is None:
             documents = history_documents(history)
         body = self.build_request(decision, user, documents)
         payload = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")

simpleaudit/targets/http.py

@@ -20,10 +20,18 @@
 from __future__ import annotations
 
 import json
+import warnings
 from typing import Any, Callable, Dict, List, Optional, Union
 
-from ..context_marks import parse_documents
-from .base import Target, TargetContext, TargetResponse, history_documents
+from ..context_marks import document_texts
+from .base import (
+    Target,
+    TargetContext,
+    TargetResponse,
+    history_documents,
+    history_file_uri,
+    plain_history,
+)
 
 
 def _resolve_path(data: Any, path: Optional[str]) -> Any:
@@ -44,24 +52,39 @@ def _resolve_path(data: Any, path: Optional[str]) -> Any:
     return cur
 
 
-#: The audit engine's own keys beside ``role`` and ``content`` (see Target.send).
-#: Documents carry evaluation-only marks, and none of them is the app's input.
-_ENGINE_MARKERS = ("documents", "file_uri", "decision", "tool_events")
-
-
 def _earlier_turns(user: str, history: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
     """The conversation before the current turn, without the engine's markers.
 
     Under ``run_scenario`` the history ends with the current user turn, which
     the body adds itself; a direct caller may pass the earlier turns only.
     Fields other than the markers (``tool_calls``, ``tool_call_id``, ...) stay.
     """
-    turns = [{k: v for k, v in m.items() if k not in _ENGINE_MARKERS} for m in history or []]
+    turns = plain_history(history)
     if turns and turns[-1].get("role") == "user" and turns[-1].get("content") == user:
         turns.pop()
     return turns
 
 
+def _with_system_text(fixed: Any, system: str) -> Any:
+    """The template's own system prompt at that field, if any, followed by the audit's."""
+    if isinstance(fixed, str) and fixed:
+        return f"{fixed}\n\n{system}"
+    if isinstance(fixed, list):  # content blocks, e.g. Anthropic's native "system"
+        return [*fixed, {"type": "text", "text": system}]
+    return system
+
+
+def _with_system_message(messages: List[Dict[str, Any]], system: str) -> List[Dict[str, Any]]:
+    """One leading system message: the template's, if it starts with one, then the audit's prompt.
+
+    Many chat templates reject two system messages in a row.
+    """
+    if messages and messages[0].get("role") == "system":
+        first = messages[0]
+        return [{**first, "content": _with_system_text(first.get("content"), system)}, *messages[1:]]
+    return [{"role": "system", "content": system}, *messages]
+
+
 class HTTPAppTarget:
     """Send audit probes to an external HTTP application and parse the answer."""
 
@@ -84,6 +107,10 @@ def __init__(
         # Optional: extract token usage from the response, e.g.
         # ("usage.prompt_tokens", "usage.completion_tokens").
         token_paths: Optional[tuple] = None,
+        # The request field for the audit's system prompt. Without it, the
+        # prompt is a leading system message in "messages" mode and cannot be
+        # sent in other modes.
+        system_field: Optional[str] = None,
     ) -> None:
         self.url = url
         self.method = method.upper()
@@ -95,12 +122,15 @@ def __init__(
         self._client = client
         self.documents_field = documents_field
         self.token_paths = token_paths
+        self.system_field = system_field
+        self._warned_system = False
 
     def _build_body(
         self,
         user: str,
         history: Optional[List[Dict[str, Any]]],
         documents: Optional[List[Union[str, Dict[str, Any]]]] = None,
+        system: Optional[str] = None,
     ) -> Dict[str, Any]:
         body = json.loads(json.dumps(self.request_template))  # deep copy
         # Retrieval context: text only, so no evaluation-only mark ever
@@ -112,7 +142,7 @@ def _build_body(
         # the context for.
         if documents is None:
             documents = history_documents(history)
-        doc_texts = [mark.text for mark in parse_documents(documents)]
+        doc_texts = document_texts(documents)
         if doc_texts:
             if self.documents_field is None:
                 raise ValueError(
@@ -122,15 +152,34 @@ def _build_body(
                 )
             body[self.documents_field] = doc_texts
         earlier = _earlier_turns(user, history)
+        if system and self.system_field:
+            body[self.system_field] = _with_system_text(body.get(self.system_field), system)
         if self.message_field == "messages":
-            # OpenAI-style chat body: ``messages`` is a list of
-            # ``{"role", "content"}`` dicts. Append the current user turn,
-            # optionally preceded by prior conversation history.
-            body["messages"] = [*earlier, {"role": "user", "content": user}]
+            # OpenAI-style chat body: any messages the request template fixes,
+            # then the conversation. Without a system_field, the audit's system
+            # prompt joins the template's leading system message or becomes one.
+            fixed = body.get("messages") if isinstance(body.get("messages"), list) else []
+            if system and not self.system_field:
+                fixed = _with_system_message(fixed, system)
+            body["messages"] = [*fixed, *earlier, {"role": "user", "content": user}]
         else:
             body[self.message_field] = user
+            if system and not self.system_field and not self._warned_system:
+                warnings.warn(
+                    "HTTPAppTarget has nowhere to send the system prompt: pass "
+                    "system_field= to name the request field for it.",
+                    stacklevel=3,
+                )
+                self._warned_system = True
             if history is not None:
-                body.setdefault("history", earlier)
+                # A history list in the template comes before the conversation.
+                fixed = body.get("history", [])
+                if not isinstance(fixed, list):
+                    raise ValueError(
+                        "request_template's 'history' must be a list for the conversation to "
+                        "follow it."
+                    )
+                body["history"] = [*fixed, *earlier]
         return body
 
     async def send(
@@ -147,10 +196,10 @@ async def send(
     ) -> TargetResponse:
         # Like documents, an attachment the app never receives must not be
         # graded as if it had.
-        if file_uri or any(m.get("file_uri") for m in history or []):
+        if file_uri or history_file_uri(history):
             raise ValueError("HTTPAppTarget cannot send attached files; this scenario attaches file_uri")
         headers = dict(self.headers)
-        body = self._build_body(user, history, documents)
+        body = self._build_body(user, history, documents, system)
 
         # Forward correlation / trace context when available.
         if context is not None:

simpleaudit/targets/openenv.py

@@ -26,7 +26,7 @@
 from dataclasses import dataclass, field
 from typing import Any
 
-from .base import TargetContext, TargetResponse
+from .base import TargetContext, TargetResponse, history_documents, history_file_uri
 
 ResetKwargs = Mapping[str, Any] | Callable[[str | None], Mapping[str, Any]]
 
@@ -171,7 +171,7 @@ async def send(
             )
         # The engine carries a scenario's attachments as markers on the
         # history's first user turn, not as send() arguments.
-        if file_uri or documents or any(m.get("file_uri") or m.get("documents") for m in history or []):
+        if file_uri or documents or history_file_uri(history) or history_documents(history):
             raise ValueError("OpenEnvTarget sends text only; this scenario attaches files or documents")
         if session.done:
             raise RuntimeError("The agent already ended this episode")