fix(criteria): search the whole command in command_executed and bump harness SDKs for Opus 5.5

.claude/notes/contracts.md

@@ -95,9 +95,14 @@ the raw text alone can stop counting.
 It is memoized because the early-stop watcher re-scans the whole accumulated trajectory on
 every tool-call event, normalizing the same command many times per run.
 
-The regex search window is capped to bound ReDoS on a large command string, and
-normalization runs over that same truncated window, so `shlex` never sees more than the cap
-and needs no separate guard.
+Each regex search is bounded to 2000 characters to limit ReDoS on a large command string,
+but the whole command is searched: the first 2000 characters, then the rest in windows that
+start and end on logical-line boundaries. Agents write long heredoc scripts that end in the
+command a task checks for (`cat > x <<EOF ... EOF` then `uip agent validate`), and a single
+leading window scored those as never run. Normalization runs per window, so `shlex` never
+sees more than the bound and needs no separate guard. Only a Bash command is windowed:
+another tool's params are JSON, and a later window of a Write/Edit body is file content,
+not a command the agent ran, so those keep the single leading window.
 
 ## Recording a CLI invocation
 

.github/workflows/harbor-e2e.yml

@@ -29,6 +29,8 @@ env:
   # Anthropic-credit spend off this path; DirectRoute is exercised elsewhere.
   API_BACKEND: "bedrock"
   CLAUDE_CODE_USE_BEDROCK: "1"
+  # Same expression as pr-checks.yml (see the comment there).
+  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }},claude-agent-sdk
 
 jobs:
   harbor-e2e:

.github/workflows/pr-checks.yml

@@ -38,11 +38,12 @@ env:
   TELEMETRY_ENABLED: "false"
   # The `uipath-*` pool enforces a minimum package-age safe-chain check on installs.
   # Workflow-level so every installing job inherits it; per-job copies are how some
-  # jobs previously ended up with no exclusions at all. The literal is the operative
-  # value — no secret of that name exists at repo or org level, so the bare `secrets.`
-  # reference this replaced resolved to an empty list. (Image builds carry their own
-  # list in docker/Dockerfile; deliberately not the same set.)
-  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }}
+  # jobs previously ended up with no exclusions at all. A repo secret of this name
+  # replaces the literal when set, so claude-agent-sdk sits outside the `||`: its wheel
+  # bundles the Claude Code CLI, and new-model support ships there faster than the
+  # gate window. (Image builds carry their own list in docker/Dockerfile; deliberately
+  # not the same set.)
+  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }},claude-agent-sdk
 
 jobs:
   quality-gate:

.github/workflows/publish-testpypi.yml

@@ -38,7 +38,7 @@ jobs:
     timeout-minutes: 10
     env:
       # `uv build` resolves build deps under the pool's safe-chain gate.
-      SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }}
+      SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }},claude-agent-sdk
     environment:
       name: testpypi
       url: https://test.pypi.org/project/coder-eval/

.github/workflows/release.yml

@@ -75,7 +75,7 @@ jobs:
       # Load-bearing on the release path: the pool enforces a package-age safe-chain
       # check on uv installs. Same expression as pr-checks.yml (see the comment there),
       # so a package can't pass PR CI and then fail the release install.
-      SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }}
+      SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }},claude-agent-sdk
 
     steps:
       # Only a real release (main) needs the app token: semantic-release pushes the

docker/Dockerfile

@@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # tag and is bumped deliberately -- mirrors the codex CLI pin
 # (`openai-codex-cli-bin==…` in pyproject). Leaving it at `@latest` let Docker
 # layer caching freeze it nondeterministically across rebuilds.
-ARG CLAUDE_CODE_VERSION=2.1.177
+ARG CLAUDE_CODE_VERSION=2.1.281
 RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
     && rm -rf /var/lib/apt/lists/* \
@@ -39,7 +39,7 @@ RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
 # Pi Node CLI, pinned — same rationale as the Claude Code pin above (the agent
 # binary is a dominant non-model driver of results; @latest would freeze
 # nondeterministically under layer caching). Node 22 + npm are already present.
-ARG PI_VERSION=0.84.4
+ARG PI_VERSION=0.87.1
 RUN npm install -g @earendil-works/pi-coding-agent@${PI_VERSION}
 
 # uv: matches host sandbox.py's `uv venv` + `uv pip install` fast path
@@ -84,12 +84,12 @@ COPY experiments/default.yaml ./experiments/default.yaml
 # pulls the `uipath` SDK from public PyPI (no credentials required).
 ARG CODER_EVAL_UV_EXTRAS=""
 
-# safe-chain min-age exclusions for the codex cli-bin pin (see pyproject [tool.uv])
-# and the pinned Antigravity harness. Codex and Antigravity are always installed,
-# so the default excludes their pinned packages from the min-age gate (the
-# google-antigravity release tracks the rapidly-moving Gemini harness and may be
+# safe-chain min-age exclusions for the codex cli-bin pin (see pyproject [tool.uv]),
+# the pinned Antigravity harness, and claude-agent-sdk (its wheel bundles the Claude
+# Code CLI). All three are always installed, so the default excludes their pinned
+# packages from the min-age gate (each tracks a rapidly-moving harness and may be
 # newer than the gate window); callers may override.
-ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex,google-antigravity"
+ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex,google-antigravity,claude-agent-sdk"
 
 # All extras (codex, antigravity, litellm, and the opt-in uipath) resolve from
 # public PyPI per uv.lock, so the build needs no private-index credentials.

docker/Dockerfile.runtime

@@ -53,7 +53,7 @@ COPY pyproject.toml uv.lock README.md ./
 COPY src/ ./src/
 COPY experiments/default.yaml ./experiments/default.yaml
 
-ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex"
+ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex,claude-agent-sdk"
 # Install the locked deps, then coder_eval itself NON-editable. An editable/path
 # install would leave the package source at /src (outside the kit), so the
 # `COPY --from /opt/coder-eval` overlay would ship a venv that can't import
@@ -66,7 +66,7 @@ RUN export SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="${SAFE_CHAIN_MINIMUM_PACKA
 
 # --- 3. Node LTS + the Claude Code CLI, under the kit dir. -------------------
 ARG NODE_VERSION=22.14.0
-ARG CLAUDE_CODE_VERSION=2.1.177
+ARG CLAUDE_CODE_VERSION=2.1.281
 # Download to a file first so a bad URL fails the build loudly (a piped
 # `curl | tar` would mask a 404 and only surface later as a missing npm), and
 # verify the tarball against nodejs.org's published SHASUMS256 before extracting
@@ -109,6 +109,6 @@ RUN chmod +x /usr/local/bin/coder_eval_entrypoint.sh
 # label; labels don't survive `COPY --from`). The claude-code pin mirrors the
 # framework image — a parity test (tests/test_image_from_dockerfiles.py) enforces it.
 ARG CODER_EVAL_VERSION=unknown
-ARG CLAUDE_CODE_VERSION=2.1.177
+ARG CLAUDE_CODE_VERSION=2.1.281
 LABEL org.coder-eval.version="${CODER_EVAL_VERSION}"
 LABEL org.coder-eval.claude-code-version="${CLAUDE_CODE_VERSION}"

docs/TASK_DEFINITION_GUIDE.md

@@ -997,6 +997,8 @@ Checks whether the agent executed specific tools/commands during evaluation. Ins
 
 **Shell normalization.** For a Bash command, both `command_pattern` and `exclude_pattern` are matched against the raw command text **and** its shell-normalized form — the `bash`/`sh`/`zsh -lc "..."` wrapper stripped and shell quoting resolved with `shlex` — and a hit on *either* form counts. So a pattern like `curated_channels` matches whether the agent wrote the argument bare, `'single'`-quoted, `"double"`-quoted, or `\"escaped\"`; you do **not** hand-encode shell quoting. Because the same haystacks also feed `exclude_pattern` and the `max_count` gate, normalization is **not** purely additive: a quote-obfuscated call can now be caught by an exclusion or a `max_count: 0` gate that the raw text alone would have missed — and, conversely, an unedited `exclude_pattern` may now exclude a call it previously let through. Cross-repo suites that hand-encoded quote tolerance in their patterns should re-baseline.
 
+**Long commands.** A Bash command is searched in full: the first 2,000 characters, then the rest in windows of at most 2,000 characters that start and end on line boundaries. So a command that follows a long heredoc in the same Bash call still counts. Other tools match only on the first 2,000 characters of their JSON parameters, so a long Write or Edit body is not read as a command. This is not purely additive either: an `exclude_pattern` or a `max_count: 0` gate can now catch a call late in a long script that it did not see before, so suites should re-baseline. A match that crosses a window boundary is not guaranteed, and `^` and `$` can also match at a window edge.
+
 **Codex limitation.** Codex agents map `Read`, `Grep`, and `Glob` tools to `shell` commands (they execute via bash), so `tool_name: "Read"` on Codex returns no matches. Use `tool_name: "Bash"` or `tool_name: null` (any tool) for Codex-compatible checks. This criterion works correctly on Claude Code agents, which emit separate `Read`/`Grep`/`Glob` telemetry.
 
 ### `cli_called`

docs/agents/ANTIGRAVITY.md

@@ -27,7 +27,7 @@ working directory — both required for an unattended eval.
 pip install 'coder-eval[antigravity]'
 ```
 
-This pulls in `google-antigravity` (pinned to `0.1.8`), whose wheel bundles the
+This pulls in `google-antigravity` (pinned to `0.1.18`), whose wheel bundles the
 platform `localharness` binary. As with the other agents the SDK is imported lazily
 — a base install without the extra still runs end-to-end; Antigravity tasks fail at
 dispatch with a clear hint to install the extra.

pyproject.toml

@@ -43,7 +43,7 @@ dependencies = [
     # explicitly since our code/tests construct httpx2 types directly. Capped to
     # mirror anthropic's own bound on it (`httpx2<3,>=2.0.0`).
     "httpx2>=2.12.0,<3.0.0",
-    "claude-agent-sdk>=0.2.124",
+    "claude-agent-sdk>=0.2.157",
     "anyio>=4.14.2",
     "radon>=6.0.1",
     "tqdm>=4.67.3",
@@ -112,7 +112,7 @@ litellm = [
 # Without this extra, the framework still installs and runs; Codex-dependent
 # code paths fail at dispatch with a clear hint pointing back here.
 codex = [
-    "openai-codex>=0.144.4",
+    "openai-codex>=0.155.1",
 ]
 # Optional extra that enables Antigravity agent support:
 #   - AntigravityAgent implementation using Google's official google-antigravity SDK
@@ -131,7 +131,7 @@ codex = [
 # Without this extra the framework still installs and runs; Antigravity-dependent
 # code paths fail at start() with a clear hint pointing back here.
 antigravity = [
-    "google-antigravity==0.1.8",
+    "google-antigravity==0.1.18",
 ]
 # Optional extra that enables OpenCode agent support.
 #
@@ -180,7 +180,7 @@ pi = []
 # back here (it is not needed to EXPORT a task to Harbor format, only to run coder-eval
 # itself as Harbor's agent).
 harbor = [
-    "harbor==0.22.0",
+    "harbor==0.23.0",
 ]
 
 [project.scripts]
@@ -212,14 +212,14 @@ packages = ["src/coder_eval"]
 allow-direct-references = true
 
 [tool.uv]
-# openai-codex 0.144.4 hardpins `openai-codex-cli-bin==0.144.4` (a stable
+# openai-codex 0.156.1 hardpins `openai-codex-cli-bin==0.156.1` (a stable
 # release; the SDK version now tracks the codex CLI version line). Naming it
 # here documents and holds the pinned cli-bin build — the harness binary is a
 # dominant non-model driver of eval results, so it travels with the coder_eval
 # release tag and is bumped deliberately (mirrors the antigravity localharness
-# and claude-code CLI pins). 0.144.4 publishes manylinux wheels (x86_64 +
+# and claude-code CLI pins). 0.156.1 publishes manylinux wheels (x86_64 +
 # aarch64), so CI/Linux installs work.
-override-dependencies = ["openai-codex-cli-bin==0.144.4"]
+override-dependencies = ["openai-codex-cli-bin==0.156.1"]
 
 constraint-dependencies = [
     # Fix known CVEs in transitive dependencies
@@ -443,6 +443,8 @@ filterwarnings = [
     "ignore::DeprecationWarning",        # Ignore deprecation warnings from dependencies
     "ignore::PendingDeprecationWarning",
     "ignore::ResourceWarning",           # Ignore unclosed resources (handled by cleanup)
+    # litellm>=1.102 TypedDicts mark fields ReadOnly; pydantic warns when it builds their schema
+    "ignore:Item '.*' on TypedDict class '.*' is using the `ReadOnly` qualifier:UserWarning",
 ]
 
 # Coverage integration (when using --cov)

src/coder_eval/agents/antigravity_agent.py

@@ -275,7 +275,7 @@ def _resolve_skills_paths(self, plugin_tools_dir: str | None) -> list[str]:
         return roots
 
     def _resolve_workspaces(self, skills_paths: list[str]) -> list[str]:
-        """Workspace roots for the harness's ``workspace_only`` file-tool policy.
+        """Workspace roots the harness confines its file tools to.
 
         The sandbox working directory (the write target) plus the resolved skill
         roots. ``skills_paths`` drives DISCOVERY only; the file-tool allowlist is
@@ -341,9 +341,9 @@ async def start(
             cfg = LocalAgentConfig(
                 model=self._effective_model(),
                 api_key=api_key,
-                # File tools are confined to ``workspaces`` by the auto-prepended
-                # workspace_only policy — see _resolve_workspaces for why the skill
-                # roots must be in here and not only in ``skills_paths``.
+                # The harness confines file tools to ``workspaces``; see
+                # _resolve_workspaces for why the skill roots must be in here and
+                # not only in ``skills_paths``.
                 workspaces=self._resolve_workspaces(skills_paths),
                 # Autonomous execution: approve every tool call, which the default
                 # policy would deny. ``permission_mode`` is deliberately NOT mapped

src/coder_eval/agents/pi_agent.py

@@ -5,7 +5,7 @@
 protocol so :class:`EventCollector` builds the ``TurnRecord``. The design mirrors
 :mod:`coder_eval.agents.opencode_agent`.
 
-Three grammar facts that are not obvious from the event names (``pi`` 0.84.4):
+Three grammar facts that are not obvious from the event names (``pi`` 0.87.1):
 
 - ``agent_start`` can appear MORE THAN ONCE per invocation — Pi auto-retries a
   transient provider error internally — and ``agent_end`` is therefore NOT
@@ -149,7 +149,7 @@
     "disallowed_tools",
 )
 
-# The full recognized Pi vocabulary (from `pi` 0.84.4). A clean exit that
+# The full recognized Pi vocabulary (from `pi` 0.87.1). A clean exit that
 # recognized NOTHING from this set is vocabulary drift and is crashed, not scored.
 # Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash
 _RECOGNIZED_EVENTS = frozenset(

src/coder_eval/criteria/command_executed.py

@@ -18,9 +18,11 @@
 
 logger = logging.getLogger(__name__)
 
-# HAZARD: bounds ReDoS on a large command string. Normalization runs over this
-# same truncated window, so shlex needs no separate size guard.
+# HAZARD: bounds ReDoS on a large command string. Every search window is at most
+# this long and normalization runs per window, so shlex needs no separate size guard.
 _MAX_PATTERN_SEARCH_LEN = 2000
+_WINDOW_OVERLAP = 256
+_LOGICAL_LINE_END = re.compile(r"(?<!\\)\n")
 
 
 def _is_shell_program(arg0: str) -> bool:
@@ -97,26 +99,60 @@ def _normalize_shell(cmd_text: str) -> str | None:
     return " ".join(tokens)
 
 
+def _search_windows(cmd_text: str) -> list[str]:
+    """Slices of ``cmd_text`` a pattern is searched in, each at most ``_MAX_PATTERN_SEARCH_LEN``.
+
+    The first window is the leading ``_MAX_PATTERN_SEARCH_LEN`` characters, so every
+    match the single leading window found is still found. The rest of the command is
+    packed into windows that start and end on logical-line boundaries (a
+    backslash-continued line stays whole), so a command on a line after a long heredoc
+    is still seen. A single logical line longer than the bound is cut into bound-sized
+    pieces overlapping by ``_WINDOW_OVERLAP``, so a match up to that long is never
+    split. Work stays linear in the command length and each regex search stays within
+    the ReDoS bound.
+    """
+    cap = _MAX_PATTERN_SEARCH_LEN
+    if len(cmd_text) <= cap:
+        return [cmd_text]
+    bounds: list[tuple[int, int]] = []
+    start = end = 0
+    for line_end in [m.end() for m in _LOGICAL_LINE_END.finditer(cmd_text)] + [len(cmd_text)]:
+        if line_end - start > cap:
+            if end > start:
+                bounds.append((start, end))
+                start = end
+            while line_end - start > cap:
+                bounds.append((start, start + cap))
+                start += cap - _WINDOW_OVERLAP
+        end = line_end
+    if end > start:
+        bounds.append((start, end))
+    return [cmd_text[:cap]] + [cmd_text[s:e] for s, e in bounds if s > 0]
+
+
 def _match_haystacks(cmd_text: str, *, is_shell: bool) -> list[str]:
     """Strings a pattern may match against for one command.
 
-    Always the raw ``cmd_text`` truncated to the ReDoS bound; when ``is_shell``,
-    additionally the quote-resolved, wrapper-stripped form of that **same
-    truncated window** (see :func:`_normalize_shell`). Normalizing the already-
-    truncated slice keeps both haystacks describing the same window, so quote-
-    stripping can never slide content from past the cap into the match, and
-    caps ``shlex`` input at ``_MAX_PATTERN_SEARCH_LEN`` for free. Matching is
-    "either" — a pattern hits the command if it matches ANY haystack.
+    When ``is_shell``, every search window of ``cmd_text`` (see
+    :func:`_search_windows`) plus the quote-resolved, wrapper-stripped form of each
+    window (see :func:`_normalize_shell`). Otherwise only the leading
+    ``_MAX_PATTERN_SEARCH_LEN`` characters: a non-shell tool's params are JSON, and a
+    later window of a Write/Edit body is file content, not something the agent ran.
+    Normalizing per window keeps every haystack within the ReDoS bound and caps
+    ``shlex`` input for free. Matching is "either" -- a pattern hits the command if it
+    matches ANY haystack.
 
     ``is_shell`` is decided once by the caller (a Bash tool whose ``command`` is
     a non-empty ``str``) and passed in, rather than re-derived here from
     ``tool_name`` alone: a Bash record with a missing/empty ``command`` serializes
     its params to JSON, where shell tokenization is meaningless, and must NOT be
     normalized (else stripped JSON quotes could newly satisfy an exclusion).
     """
-    window = cmd_text[:_MAX_PATTERN_SEARCH_LEN]
-    haystacks = [window]
-    if is_shell:
+    if not is_shell:
+        return [cmd_text[:_MAX_PATTERN_SEARCH_LEN]]
+    haystacks: list[str] = []
+    for window in _search_windows(cmd_text):
+        haystacks.append(window)
         normalized = _normalize_shell(window)
         if normalized is not None and normalized != window:
             haystacks.append(normalized)

src/coder_eval/harbor/agent.py

@@ -20,7 +20,7 @@
 from __future__ import annotations
 
 import shlex
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ClassVar
 
 from coder_eval.harbor.agent_paths import AGENT_TASK_YAML_PATH
 
@@ -62,7 +62,7 @@ class CoderEvalAgent(BaseInstalledAgent):
     on PATH there exactly as ``tests/test.sh`` assumes it is for grading.
     """
 
-    SUPPORTS_ATIF: bool = True
+    SUPPORTS_ATIF: ClassVar[bool] = True
 
     @staticmethod
     def name() -> str: