From 5fc99fd09eaa4078e0f7956977f3a4e66d12c581 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Fri, 7 Aug 2026 16:30:23 +0300 Subject: [PATCH 1/5] fix(agent): append system_prompt to the Claude Code preset instead of replacing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plain-string ClaudeAgentOptions.system_prompt replaces Claude Code's entire default system prompt. Every experiment that sets even a one-line system_prompt silently strips the harness's behavioral guidance — observed in skills nightly runs as zero parallel tool calls (the batching instruction lives in the default prompt), heavy narration, and raw cat/sed over Read/Grep. Wrap the configured prompt in the SDK's claude_code preset with append so the default prompt survives. Co-Authored-By: Claude Fable 5 --- src/coder_eval/agents/claude_code_agent.py | 11 ++++++++- src/coder_eval/models/agent_config.py | 3 ++- tests/test_agent.py | 26 ++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 71cb2267..cfd5bc9a 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -20,6 +20,7 @@ TaskNotificationMessage, query, ) +from claude_agent_sdk.types import SystemPromptPreset # Private SDK import — the public `query()` API doesn't expose the subprocess # handle, but we need it to SIGKILL on timeout (the SDK's anyio task groups @@ -1173,6 +1174,14 @@ def _build_claude_query( if "ToolSearch" not in disallowed_tools: disallowed_tools.append("ToolSearch") + # A plain-string system_prompt would REPLACE Claude Code's default system + # prompt, dropping its behavioral guidance (parallel tool-call batching, + # conciseness). Always keep the default via the SDK preset and append the + # configured prompt after it. + system_prompt: SystemPromptPreset | None = None + if self.config.system_prompt is not None: + system_prompt = SystemPromptPreset(type="preset", preset="claude_code", append=self.config.system_prompt) + # as_posix(), not str(): bash on Windows strips backslashes from unquoted # paths, so a redirect like `> D:\foo\bar` ends up writing to "Dfoobar". options = ClaudeAgentOptions( @@ -1192,7 +1201,7 @@ def _build_claude_query( # summing per-message values undercounts by 10x+. Without this flag # StreamEvents are suppressed by the SDK. include_partial_messages=True, - system_prompt=self.config.system_prompt, + system_prompt=system_prompt, setting_sources=self.config.setting_sources if self.config.setting_sources is not None else ["project"], resume=self._session_id, settings=json.dumps(self.config.claude_settings) diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index b4ad98fd..b35d286d 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -151,7 +151,8 @@ class BaseAgentConfig(BaseModel): system_prompt: str | None = Field( default=None, description=( - "Custom system prompt. Replaces the default system prompt. " + "Custom system prompt, appended to the agent's default system prompt " + "(claude-code: the SDK 'claude_code' preset with append). " "Supports inline text or multi-line YAML strings. " "Mutually exclusive with system_prompt_file." ), diff --git a/tests/test_agent.py b/tests/test_agent.py index 2e4f7aaa..405d2996 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -377,6 +377,32 @@ async def test_claude_settings_none_default(): assert captured_options[0].settings is None +@pytest.mark.asyncio +async def test_system_prompt_appends_to_claude_code_preset(): + """system_prompt keeps the Claude Code default prompt and appends via the SDK preset.""" + config = parse_agent_config(type=AgentKind.CLAUDE_CODE, system_prompt="You are a coding agent.") + agent = ClaudeCodeAgent(config) + + captured_options = await _capture_sdk_options(agent) + + assert captured_options[0].system_prompt == { + "type": "preset", + "preset": "claude_code", + "append": "You are a coding agent.", + } + + +@pytest.mark.asyncio +async def test_system_prompt_none_leaves_sdk_default(): + """No system_prompt -> ClaudeAgentOptions.system_prompt stays None (SDK default prompt).""" + config = parse_agent_config(type=AgentKind.CLAUDE_CODE) + agent = ClaudeCodeAgent(config) + + captured_options = await _capture_sdk_options(agent) + + assert captured_options[0].system_prompt is None + + @pytest.mark.asyncio async def test_sdk_options_forwarded_to_sdk(): """An sdk_options key (e.g. effort) is splatted into ClaudeAgentOptions.""" From f0668343cb7d300c5c68a34ed59a31b3a0bd190e Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Fri, 7 Aug 2026 16:35:15 +0300 Subject: [PATCH 2/5] style: sort claude_agent_sdk.types import Co-Authored-By: Claude Fable 5 --- src/coder_eval/agents/claude_code_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index cfd5bc9a..208cc574 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -20,7 +20,6 @@ TaskNotificationMessage, query, ) -from claude_agent_sdk.types import SystemPromptPreset # Private SDK import — the public `query()` API doesn't expose the subprocess # handle, but we need it to SIGKILL on timeout (the SDK's anyio task groups @@ -28,6 +27,7 @@ # CLI). If this import breaks on an SDK upgrade, the threaded watchdog loses # its kill target and timeouts will no longer be enforced at the agent layer. from claude_agent_sdk._internal.transport.subprocess_cli import SubprocessCLITransport +from claude_agent_sdk.types import SystemPromptPreset from coder_eval.agent import Agent, AgentState from coder_eval.agents._logging import PrefixedAdapter, log_raw_sdk_event From b0113d36bb7782431f7e0a655b33cce9a7e8c19f Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 8 Aug 2026 09:36:53 +0300 Subject: [PATCH 3/5] fix(agent): align Codex system_prompt with the append-only contract CodexAgent silently dropped config.system_prompt; forward it as developer_instructions (injected on top of the Codex base prompt) to match the append semantics of Claude Code (claude_code preset) and Antigravity (TemplatedSystemInstructions, which already appended). Also document the ripple effects of append-only system_prompt: - agent_judge: the reviewer prompt is now layered after the full Claude Code preset instead of replacing it (accepted trade-off, noted in code) - BaseAgentConfig.system_prompt description states per-agent semantics - docs: fix the stale "Replaces the default" claim in CLAUDE_CODE.md, add a System prompt row to CODEX.md, document Antigravity's append shorthand Co-Authored-By: Claude Fable 5 --- docs/agents/ANTIGRAVITY.md | 8 ++++++++ docs/agents/CLAUDE_CODE.md | 2 +- docs/agents/CODEX.md | 1 + src/coder_eval/agents/codex_agent.py | 8 ++++++++ src/coder_eval/criteria/agent_judge.py | 6 ++++++ src/coder_eval/models/agent_config.py | 5 +++-- tests/test_codex_agent.py | 16 ++++++++++++++++ 7 files changed, 43 insertions(+), 3 deletions(-) diff --git a/docs/agents/ANTIGRAVITY.md b/docs/agents/ANTIGRAVITY.md index 522dc8bf..ad96081b 100644 --- a/docs/agents/ANTIGRAVITY.md +++ b/docs/agents/ANTIGRAVITY.md @@ -109,6 +109,14 @@ Antigravity exposes a `thinking_level` field (`minimal` / `low` / `medium` / Antigravity-specific — Claude Code and Codex don't take this field. Thinking tokens are billed as **output** tokens (see [Telemetry](#telemetry)). +### `system_prompt` + +`agent.system_prompt` is passed to the SDK as `system_instructions`, whose string +shorthand maps to `TemplatedSystemInstructions` — a named section **appended** to +the harness's default system instructions, never a replacement. This matches the +append-only semantics of the shared config field across agents (Claude Code appends +via the `claude_code` preset; Codex via `developer_instructions`). + ### Skills (SKILL.md) Antigravity supports [Agent Skills](https://agentskills.io/specification) diff --git a/docs/agents/CLAUDE_CODE.md b/docs/agents/CLAUDE_CODE.md index 66670709..5614b499 100644 --- a/docs/agents/CLAUDE_CODE.md +++ b/docs/agents/CLAUDE_CODE.md @@ -99,7 +99,7 @@ agent: | `allowed_tools` | `list[str] \| null` | Tool allowlist. Unset ⇒ all tools allowed. | | `disallowed_tools` | `list[str] \| null` | Tool denylist. (`ToolSearch` is always appended for Bedrock parity.) | | `plugins` | `list[{type: local, path}]` | Local plugin/skill directories; `$VAR` in `path` is expanded and resolved to an absolute path. | -| `system_prompt` | `str \| null` | **Replaces** the default system prompt (there is no *append* seam). Mutually exclusive with `system_prompt_file`. | +| `system_prompt` | `str \| null` | **Appended** to the default Claude Code system prompt (via the SDK's `claude_code` preset with `append`) — the default's behavioral guidance is always kept. Mutually exclusive with `system_prompt_file`. | | `system_prompt_file` | `str \| null` | Path (relative to the task YAML) loaded into `system_prompt` at resolution. | | `setting_sources` | `list["user"\|"project"\|"local"] \| null` | Which host setting sources the SDK reads. Default resolves to `["project"]`. See [Sandbox isolation](#sandbox-isolation). | | `claude_settings` | `str \| dict \| null` | Passed to the SDK `--settings`. A dict is JSON-serialized; a str is a settings file path. Use `permissions.deny` to block tools/paths. | diff --git a/docs/agents/CODEX.md b/docs/agents/CODEX.md index b020e76b..7b6dc99f 100644 --- a/docs/agents/CODEX.md +++ b/docs/agents/CODEX.md @@ -213,6 +213,7 @@ The Codex SDK is synchronous. The agent uses `_run_async()` helper to detect and | **SDK Type** | Subprocess (CLI via JSON generator) | Sync client (app-server subprocess) | | **Command Tracking** | Full telemetry (tool name, params, duration) | Streamed telemetry: shell → `Bash`, apply_patch → `Write` | | **Model Selection** | Direct via `--model` or config | `agent.model` pinned into `thread_start` | +| **System prompt** | `system_prompt` appended to the default prompt (SDK `claude_code` preset) | `system_prompt` passed as `developer_instructions` on top of the Codex base prompt | | **Session Resume** | `--resume {session_id}` | Via thread ID | | **Permissions** | `permission_mode` + `allowed_tools` | `permission_mode` → sandbox/approval + `allowed_tools`/`disallowed_tools` → thread config | | **Tool Enforcement** | Not enforced by Coder Eval wrapper | `enabled_tools` honored; `disabled_tools` NOT enforced by the SDK | diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index c6e4b2d3..4b3af0d0 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -1276,6 +1276,14 @@ def _build_thread_options(self) -> dict[str, Any]: options["model"] = effective_model self._log.debug(f"Codex model pinned to {effective_model}") + # system_prompt maps to developer_instructions: injected ON TOP of Codex's + # base prompt, matching the append-only contract of the shared config field + # (Claude Code appends via the claude_code preset; Antigravity via + # TemplatedSystemInstructions). base_instructions (full replacement of the + # base prompt) is deliberately not exposed. + if self.config.system_prompt is not None: + options["developer_instructions"] = self.config.system_prompt + permission_mode = self.config.permission_mode.value approval_mode_str = _CODEX_APPROVAL_MODE diff --git a/src/coder_eval/criteria/agent_judge.py b/src/coder_eval/criteria/agent_judge.py index 20db8dab..3af1fea4 100644 --- a/src/coder_eval/criteria/agent_judge.py +++ b/src/coder_eval/criteria/agent_judge.py @@ -63,6 +63,12 @@ logger = logging.getLogger(__name__) +# system_prompt is append-only (see BaseAgentConfig.system_prompt), so this is +# layered AFTER the full Claude Code preset rather than replacing it: the judge +# carries the coding-agent identity plus this reviewer role, and pays the +# preset's prompt tokens on every call. Accepted trade-off — the preset's +# tool-usage guidance helps the investigation, and the submit_verdict contract +# below still governs the output. _SYSTEM_PROMPT = """\ You are a strict code reviewer evaluating a project generated by a coding agent. diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index b35d286d..8f389798 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -151,8 +151,9 @@ class BaseAgentConfig(BaseModel): system_prompt: str | None = Field( default=None, description=( - "Custom system prompt, appended to the agent's default system prompt " - "(claude-code: the SDK 'claude_code' preset with append). " + "Custom system prompt, appended to the agent's default system prompt — never a replacement " + "(claude-code: the SDK 'claude_code' preset with append; codex: developer_instructions " + "on top of the base prompt; antigravity: TemplatedSystemInstructions sections). " "Supports inline text or multi-line YAML strings. " "Mutually exclusive with system_prompt_file." ), diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index 95ee117f..f9f009dc 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -125,6 +125,22 @@ def test_sandbox_is_full_access(self, monkeypatch, mode, in_container, os_name): assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") +class TestSystemPrompt: + """system_prompt travels as developer_instructions — injected on top of Codex's + base prompt, mirroring the append-only semantics of the other agents.""" + + def test_system_prompt_forwarded_as_developer_instructions(self): + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, system_prompt="You are a coding agent.")) + + assert agent._build_thread_options()["developer_instructions"] == "You are a coding agent." + + def test_no_system_prompt_omits_developer_instructions(self): + """No system_prompt -> the key is absent, leaving the SDK default untouched.""" + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX)) + + assert "developer_instructions" not in agent._build_thread_options() + + class TestCodexEnvironmentConfiguration: """Test _build_codex_env: only CODEX_API_KEY travels via env.""" From 92ebce9b96ba612053bb734b23614bb9b2e6d871 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 8 Aug 2026 10:00:00 +0300 Subject: [PATCH 4/5] =?UTF-8?q?fix(agent):=20address=20PR=20review=20?= =?UTF-8?q?=E2=80=94=20unconditional=20preset,=20judge=20replace=20seam?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blockers from the PR #92 review: - system_prompt unset no longer loses the preset: the SDK maps None to --system-prompt "" (an explicit EMPTY prompt), so _build_options now always sends the claude_code preset — bare (CLI default prompt) when unset, with `append` when configured. This fixes the common no- system_prompt case, which previously ran without Claude Code's default behavioral guidance. - agent_judge no longer inherits the coding-agent preset: new ClaudeCodeAgentConfig.system_prompt_mode ("append" default / "replace"), forced to "replace" in _build_agent_config next to the existing security floors, so the judge prompt stays its entire identity and verdicts can't shift with the preset. Pinned by test. - exclude_dynamic_sections=True on the preset keeps the system prompt static across runs (no per-run tempdir path baked in); the SDK re-injects the stripped sections into the first user message. - Transport-level tests: captured options are rendered through SubprocessCLITransport._build_command() asserting the exact flag emitted (--append-system-prompt vs --system-prompt vs none) — the surface the original bug lived on. Also pins system_prompt: "" and the renamed unset-case test (the old name asserted a false SDK contract). - BaseAgentConfig.system_prompt description is agent-neutral again; the claude-specific mechanism lives on ClaudeCodeAgentConfig + docs/agents/. MIGRATION NOTE: system_prompt semantics on claude-code changed from replace to append, and runs WITHOUT system_prompt now get the real Claude Code default prompt instead of an empty one. Scores are comparable only within one semantics regime — re-baseline judged tasks (e.g. tasks/python_cli_simulated_judged/echo_simulated_judged.yaml, whose prompt was written against replace semantics) and pin runs to the CLI version recorded in environment_info.claude_code_cli. Co-Authored-By: Claude Fable 5 --- docs/agents/CLAUDE_CODE.md | 12 ++++- src/coder_eval/agents/claude_code_agent.py | 28 +++++++--- src/coder_eval/criteria/agent_judge.py | 15 +++--- src/coder_eval/models/agent_config.py | 14 +++-- tests/test_agent.py | 61 ++++++++++++++++++++-- tests/test_agent_judge_criterion.py | 16 ++++++ 6 files changed, 126 insertions(+), 20 deletions(-) diff --git a/docs/agents/CLAUDE_CODE.md b/docs/agents/CLAUDE_CODE.md index 5614b499..c752f15b 100644 --- a/docs/agents/CLAUDE_CODE.md +++ b/docs/agents/CLAUDE_CODE.md @@ -99,7 +99,8 @@ agent: | `allowed_tools` | `list[str] \| null` | Tool allowlist. Unset ⇒ all tools allowed. | | `disallowed_tools` | `list[str] \| null` | Tool denylist. (`ToolSearch` is always appended for Bedrock parity.) | | `plugins` | `list[{type: local, path}]` | Local plugin/skill directories; `$VAR` in `path` is expanded and resolved to an absolute path. | -| `system_prompt` | `str \| null` | **Appended** to the default Claude Code system prompt (via the SDK's `claude_code` preset with `append`) — the default's behavioral guidance is always kept. Mutually exclusive with `system_prompt_file`. | +| `system_prompt` | `str \| null` | **Appended** to the default Claude Code system prompt (via the SDK's `claude_code` preset) — the default's behavioral guidance is always kept, whether or not this is set. Mutually exclusive with `system_prompt_file`. | +| `system_prompt_mode` | `"append"` (default) / `"replace"` | `replace` sends `system_prompt` as the **entire** system prompt (no preset). Used by judge sub-agents, which must not carry the coding-agent persona; rarely needed in tasks. | | `system_prompt_file` | `str \| null` | Path (relative to the task YAML) loaded into `system_prompt` at resolution. | | `setting_sources` | `list["user"\|"project"\|"local"] \| null` | Which host setting sources the SDK reads. Default resolves to `["project"]`. See [Sandbox isolation](#sandbox-isolation). | | `claude_settings` | `str \| dict \| null` | Passed to the SDK `--settings`. A dict is JSON-serialized; a str is a settings file path. Use `permissions.deny` to block tools/paths. | @@ -111,6 +112,15 @@ agent: > `setting_sources`, `include_partial_messages`, …) are rejected there — set those > through their typed fields or `-D run_limits.*`. MCP servers are not a YAML field. +> **System-prompt reproducibility.** In `append` mode the preset's *dynamic +> sections* (working directory, git status, auto-memory) are excluded so the system +> prompt stays identical across runs — the per-run sandbox tempdir path would +> otherwise be baked into it, breaking prompt caching and run comparability. The +> SDK re-injects the stripped content into the first user message, so the agent +> loses nothing. Note the default-prompt baseline tracks the installed Claude Code +> CLI version; `environment_info.claude_code_cli` in `run.json` records which +> version a run used. + ### Setting fields from the CLI Any of these merge-resolve through `-D` / `--set` (see diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 208cc574..6dccda46 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -27,6 +27,9 @@ # CLI). If this import breaks on an SDK upgrade, the threaded watchdog loses # its kill target and timeouts will no longer be enforced at the agent layer. from claude_agent_sdk._internal.transport.subprocess_cli import SubprocessCLITransport + +# SystemPromptPreset is not re-exported from the SDK root, so claude_agent_sdk.types +# is the only import route (same treatment as evaluation/verdict_tool.py). from claude_agent_sdk.types import SystemPromptPreset from coder_eval.agent import Agent, AgentState @@ -1174,13 +1177,24 @@ def _build_claude_query( if "ToolSearch" not in disallowed_tools: disallowed_tools.append("ToolSearch") - # A plain-string system_prompt would REPLACE Claude Code's default system - # prompt, dropping its behavioral guidance (parallel tool-call batching, - # conciseness). Always keep the default via the SDK preset and append the - # configured prompt after it. - system_prompt: SystemPromptPreset | None = None - if self.config.system_prompt is not None: - system_prompt = SystemPromptPreset(type="preset", preset="claude_code", append=self.config.system_prompt) + # The SDK maps system_prompt=None to `--system-prompt ""` (an explicit + # EMPTY custom prompt) and a plain string to a full replacement — either + # way Claude Code's default behavioral guidance (parallel tool-call + # batching, conciseness) is lost. So ALWAYS send the claude_code preset: + # without `append` the CLI runs its default prompt; with it the configured + # prompt is appended. exclude_dynamic_sections keeps the prompt static + # across runs (the per-run tempdir path would otherwise be baked into the + # system prompt, breaking prompt caching and run comparability); the SDK + # re-injects the stripped sections into the first user message. + # system_prompt_mode="replace" (judge sub-agents) opts out of the preset: + # the configured prompt IS the entire system prompt. + system_prompt: str | SystemPromptPreset + if self.config.system_prompt_mode == "replace" and self.config.system_prompt is not None: + system_prompt = self.config.system_prompt + else: + system_prompt = SystemPromptPreset(type="preset", preset="claude_code", exclude_dynamic_sections=True) + if self.config.system_prompt is not None: + system_prompt["append"] = self.config.system_prompt # as_posix(), not str(): bash on Windows strips backslashes from unquoted # paths, so a redirect like `> D:\foo\bar` ends up writing to "Dfoobar". diff --git a/src/coder_eval/criteria/agent_judge.py b/src/coder_eval/criteria/agent_judge.py index 3af1fea4..bbbc6e48 100644 --- a/src/coder_eval/criteria/agent_judge.py +++ b/src/coder_eval/criteria/agent_judge.py @@ -63,12 +63,11 @@ logger = logging.getLogger(__name__) -# system_prompt is append-only (see BaseAgentConfig.system_prompt), so this is -# layered AFTER the full Claude Code preset rather than replacing it: the judge -# carries the coding-agent identity plus this reviewer role, and pays the -# preset's prompt tokens on every call. Accepted trade-off — the preset's -# tool-usage guidance helps the investigation, and the submit_verdict contract -# below still governs the output. +# This is the judge's ENTIRE identity: _build_agent_config forces +# system_prompt_mode="replace" so the Claude Code coding-agent preset never +# reaches the scoring instrument — the judge must not carry an engineering +# persona (terse, proactively edits files) ahead of its grading role, and its +# verdicts must not shift when the preset does. _SYSTEM_PROMPT = """\ You are a strict code reviewer evaluating a project generated by a coding agent. @@ -269,6 +268,10 @@ def _build_agent_config( user_overrides["sdk_options"] = {**defaults.sdk_options, **user_overrides["sdk_options"]} config = defaults.model_copy(update=user_overrides, deep=True) config.system_prompt = system_prompt + # Force replace regardless of user YAML: the judge prompt is its entire + # identity — the coding-agent preset must never prefix the scoring + # instrument (see the note on _SYSTEM_PROMPT). + config.system_prompt_mode = "replace" # SECURITY: force setting_sources=[] regardless of user YAML so the SDK # does NOT load .claude/settings.json or .mcp.json from the judge's cwd. # Those files can install pre-LLM lifecycle hooks (SessionStart / diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index 8f389798..721997c1 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -151,9 +151,8 @@ class BaseAgentConfig(BaseModel): system_prompt: str | None = Field( default=None, description=( - "Custom system prompt, appended to the agent's default system prompt — never a replacement " - "(claude-code: the SDK 'claude_code' preset with append; codex: developer_instructions " - "on top of the base prompt; antigravity: TemplatedSystemInstructions sections). " + "Custom system prompt, appended to the agent's default system prompt — never a replacement. " + "Each agent's doc page (docs/agents/) states the exact mechanism. " "Supports inline text or multi-line YAML strings. " "Mutually exclusive with system_prompt_file." ), @@ -199,6 +198,15 @@ class ClaudeCodeAgentConfig(BaseAgentConfig): type: Literal[AgentKind.CLAUDE_CODE] # type: ignore[assignment] + system_prompt_mode: Literal["append", "replace"] = Field( + default="append", + description=( + "How system_prompt combines with the Claude Code default prompt: 'append' layers it " + "after the SDK 'claude_code' preset, keeping the default's behavioral guidance; " + "'replace' sends it as the ENTIRE system prompt. Judge sub-agents force 'replace' so " + "the scoring instrument never carries the coding-agent persona." + ), + ) claude_settings: str | dict[str, Any] | None = MergeField( strategy="deep", default=None, diff --git a/tests/test_agent.py b/tests/test_agent.py index 405d2996..59aa019e 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -377,6 +377,19 @@ async def test_claude_settings_none_default(): assert captured_options[0].settings is None +def _transport_command(options) -> list[str]: + """Render captured ClaudeAgentOptions into the actual CLI argv. + + The dict-shape assertions pin the values we set; this pins the SDK contract + (which flag the transport emits) — the surface the original replace-vs-append + bug lived on — and survives an SDK TypedDict reshape. + """ + from claude_agent_sdk._internal.transport.subprocess_cli import SubprocessCLITransport + + options.cli_path = "claude" + return SubprocessCLITransport(prompt="x", options=options)._build_command() + + @pytest.mark.asyncio async def test_system_prompt_appends_to_claude_code_preset(): """system_prompt keeps the Claude Code default prompt and appends via the SDK preset.""" @@ -388,19 +401,61 @@ async def test_system_prompt_appends_to_claude_code_preset(): assert captured_options[0].system_prompt == { "type": "preset", "preset": "claude_code", + "exclude_dynamic_sections": True, "append": "You are a coding agent.", } + cmd = _transport_command(captured_options[0]) + assert "--append-system-prompt" in cmd + assert "--system-prompt" not in cmd @pytest.mark.asyncio -async def test_system_prompt_none_leaves_sdk_default(): - """No system_prompt -> ClaudeAgentOptions.system_prompt stays None (SDK default prompt).""" +async def test_system_prompt_unset_sends_bare_preset(): + """No system_prompt -> the bare claude_code preset, which the transport renders + as NO system-prompt flag (the CLI default). Passing None instead would emit + `--system-prompt \"\"` — an explicit EMPTY prompt that loses the default.""" config = parse_agent_config(type=AgentKind.CLAUDE_CODE) agent = ClaudeCodeAgent(config) captured_options = await _capture_sdk_options(agent) - assert captured_options[0].system_prompt is None + assert captured_options[0].system_prompt == { + "type": "preset", + "preset": "claude_code", + "exclude_dynamic_sections": True, + } + cmd = _transport_command(captured_options[0]) + assert "--append-system-prompt" not in cmd + assert "--system-prompt" not in cmd + + +@pytest.mark.asyncio +async def test_system_prompt_empty_string_appends_empty(): + """system_prompt: \"\" is configured, not unset — it appends (harmlessly), and a + future truthiness refactor must not route it into the preset-loss path.""" + config = parse_agent_config(type=AgentKind.CLAUDE_CODE, system_prompt="") + agent = ClaudeCodeAgent(config) + + captured_options = await _capture_sdk_options(agent) + + assert captured_options[0].system_prompt["append"] == "" + + +@pytest.mark.asyncio +async def test_system_prompt_mode_replace_sends_plain_string(): + """system_prompt_mode='replace' (the judge seam) sends the configured prompt as + the ENTIRE system prompt — no preset, no coding-agent persona.""" + config = parse_agent_config( + type=AgentKind.CLAUDE_CODE, system_prompt="You are a strict grader.", system_prompt_mode="replace" + ) + agent = ClaudeCodeAgent(config) + + captured_options = await _capture_sdk_options(agent) + + assert captured_options[0].system_prompt == "You are a strict grader." + cmd = _transport_command(captured_options[0]) + assert "--system-prompt" in cmd + assert "--append-system-prompt" not in cmd @pytest.mark.asyncio diff --git a/tests/test_agent_judge_criterion.py b/tests/test_agent_judge_criterion.py index cfd6b720..593764bc 100644 --- a/tests/test_agent_judge_criterion.py +++ b/tests/test_agent_judge_criterion.py @@ -715,6 +715,22 @@ def test_agent_judge_prompt_requires_findings(sandbox: Sandbox, direct_route: Di assert "findings" in user_msg.lower() +def test_agent_judge_system_prompt_replaces_not_appends(sandbox: Sandbox, direct_route: DirectRoute) -> None: + """The judge prompt is its ENTIRE identity: system_prompt_mode must be 'replace' + so the Claude Code coding-agent preset never prefixes the scoring instrument — + forced even when the user's YAML says 'append'.""" + criterion = AgentJudgeCriterion( + description="x", prompt="grade", agent={"type": "claude-code", "system_prompt_mode": "append"} + ) + mock_agent = _make_mock_agent('{"score": 0.5, "rationale": "ok"}') + with patch(_AGENT_PATCH_PATH, return_value=mock_agent) as mock_cls: + SuccessChecker(sandbox, init_registry=False, route=direct_route).check(criterion) + + (agent_config,) = mock_cls.call_args.args + assert agent_config.system_prompt_mode == "replace" + assert agent_config.system_prompt.startswith("You are a strict code reviewer") + + def test_agent_judge_transcript_captures_tool_calls(sandbox: Sandbox, direct_route: DirectRoute) -> None: """Tool calls made by the judge sub-agent must surface on the transcript so reviewers can audit the verdict.""" From 3083fc87b1773921ac1a0339a0a41ec10b433b53 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 8 Aug 2026 10:19:50 +0300 Subject: [PATCH 5/5] feat(agent): record system_prompt_semantics marker in environment_info Trend dashboards need to segment runs by system-prompt regime instead of silently pooling pre-/post-append-semantics scores (PR #92 review, cross-run comparability blocker). Each built-in agent now emits system_prompt_semantics via get_environment_info(), merged into run.json: - claude-code: the resolved system_prompt_mode ("append" / "replace") - codex: "append" (developer_instructions; previously the field was silently dropped, so codex runs also cross a semantics boundary here) - antigravity: "append" (unchanged behavior, emitted for uniformity) Runs without the marker predate the change and used replace-on-set / empty-on-unset (claude-code) or dropped (codex) semantics. Co-Authored-By: Claude Fable 5 --- docs/agents/CLAUDE_CODE.md | 4 +++- src/coder_eval/agents/antigravity_agent.py | 3 +++ src/coder_eval/agents/claude_code_agent.py | 11 +++++++++++ src/coder_eval/agents/codex_agent.py | 8 +++++++- tests/test_agent.py | 13 +++++++++++++ tests/test_antigravity_agent.py | 8 ++++++++ tests/test_codex_agent.py | 7 +++++-- 7 files changed, 50 insertions(+), 4 deletions(-) diff --git a/docs/agents/CLAUDE_CODE.md b/docs/agents/CLAUDE_CODE.md index c752f15b..8b7aab0d 100644 --- a/docs/agents/CLAUDE_CODE.md +++ b/docs/agents/CLAUDE_CODE.md @@ -119,7 +119,9 @@ agent: > SDK re-injects the stripped content into the first user message, so the agent > loses nothing. Note the default-prompt baseline tracks the installed Claude Code > CLI version; `environment_info.claude_code_cli` in `run.json` records which -> version a run used. +> version a run used, and `environment_info.system_prompt_semantics` +> (`append` / `replace`) records the prompt regime — runs predating that marker +> used replace-on-set / empty-on-unset semantics and are not score-comparable. ### Setting fields from the CLI diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 4e1475be..9c1d58ce 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -561,6 +561,9 @@ def get_environment_info(self) -> dict[str, Any]: return { "antigravity_model": self._effective_model(), "antigravity_thinking_level": self.config.thinking_level, + # Antigravity has always appended (TemplatedSystemInstructions); + # emitted for cross-agent uniformity of the marker. + "system_prompt_semantics": "append", } def _conversation_or_none(self) -> Any: diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 6dccda46..3671fa7f 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -1239,6 +1239,17 @@ def _build_claude_query( return options, transport, effective_model + def get_environment_info(self) -> dict[str, Any]: + """Record which system-prompt regime built this run's prompts. + + ``append`` = the claude_code preset (dynamic sections excluded) with the + configured system_prompt, if any, appended; ``replace`` = the configured + prompt is the ENTIRE system prompt (judge sub-agents). Runs from before + this marker existed used replace-on-set / empty-on-unset semantics — + trend dashboards must not pool scores across that boundary. + """ + return {"system_prompt_semantics": self.config.system_prompt_mode} + async def stop(self) -> None: """Stop the agent and clean up resources.""" self.client = None diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 4b3af0d0..e3f40f7d 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -944,10 +944,16 @@ def get_environment_info(self) -> dict[str, Any]: recorded to avoid leaking any embedded credentials; the API key is never recorded. """ + # system_prompt_semantics: Codex appends system_prompt as + # developer_instructions on top of its base prompt. Runs from before this + # marker existed silently DROPPED the field — dashboards must not pool + # system_prompt-setting tasks across that boundary. + info: dict[str, Any] = {"system_prompt_semantics": "append"} base_url = self._resolve_base_url() if not base_url: - return {} + return info return { + **info, "codex_base_url_host": urlparse(base_url).hostname or "", "codex_wire_api": _CODEX_WIRE_API, "codex_api_version": self._resolve_api_version() or "", diff --git a/tests/test_agent.py b/tests/test_agent.py index 59aa019e..1005a8e9 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -458,6 +458,19 @@ async def test_system_prompt_mode_replace_sends_plain_string(): assert "--append-system-prompt" not in cmd +def test_environment_info_reports_system_prompt_semantics(): + """The resolved system_prompt_mode lands in run.json (environment_info) so + trend dashboards can segment runs by prompt regime instead of pooling + pre-/post-append-semantics scores.""" + default_agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE)) + assert default_agent.get_environment_info() == {"system_prompt_semantics": "append"} + + judge_like = ClaudeCodeAgent( + parse_agent_config(type=AgentKind.CLAUDE_CODE, system_prompt="grader", system_prompt_mode="replace") + ) + assert judge_like.get_environment_info() == {"system_prompt_semantics": "replace"} + + @pytest.mark.asyncio async def test_sdk_options_forwarded_to_sdk(): """An sdk_options key (e.g. effort) is splatted into ClaudeAgentOptions.""" diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index 99747950..0f989b07 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -64,6 +64,14 @@ def test_effective_model_prefers_config_then_default(): assert unpinned._effective_model() == _DEFAULT_MODEL +def test_environment_info_reports_append_prompt_semantics(): + """Antigravity always appends system_prompt (TemplatedSystemInstructions); + the cross-agent marker in run.json records that regime.""" + agent = AntigravityAgent(parse_agent_config(type="antigravity")) + + assert agent.get_environment_info()["system_prompt_semantics"] == "append" + + def _make_skill(parent, name: str) -> None: d = parent / name d.mkdir(parents=True) diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index f9f009dc..33690b0a 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -331,10 +331,12 @@ def test_empty_api_version_falls_back(self, monkeypatch): class TestCodexEnvironmentInfo: """get_environment_info surfaces resolved custom-endpoint routing for run artifacts.""" - def test_no_base_url_emits_nothing(self, monkeypatch): + def test_no_base_url_emits_only_prompt_semantics(self, monkeypatch): + """Without a custom endpoint, only the cross-agent system-prompt marker is + emitted (Codex appends system_prompt as developer_instructions).""" monkeypatch.delenv("CODEX_BASE_URL", raising=False) agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, model="gpt-5-codex")) - assert agent.get_environment_info() == {} + assert agent.get_environment_info() == {"system_prompt_semantics": "append"} def test_azure_routing_recorded(self, monkeypatch): """Host (not full URL), wire_api, api-version, and the deployment-name marker @@ -345,6 +347,7 @@ def test_azure_routing_recorded(self, monkeypatch): agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, model="my-gpt5-deployment")) info = agent.get_environment_info() assert info == { + "system_prompt_semantics": "append", "codex_base_url_host": "my-res.openai.azure.com", "codex_wire_api": "responses", "codex_api_version": "2025-04-01-preview",