fix(docker): harness isolation via COPY/PRUNE + GRADE-OUTSIDE. Close the criteria/grader leak

.github/workflows/pr-checks.yml

@@ -405,6 +405,18 @@ jobs:
       - name: Build coder-eval-agent base Docker image
         run: make docker-image
 
+      # Docker isolation detectors (COPY/PRUNE + GRADE-OUTSIDE). The daemon-less
+      # set is the load-bearing CI sensor: host-unchanged proxy (no rw host-original
+      # mount) + criteria absence + baked-image scan + no-uid-drop-machinery guard.
+      - name: Run docker isolation detectors (daemon-less)
+        run: make test-docker-detectors
+
+      # Exit-criterion sensor: a real docker run must leave the host byte-for-byte
+      # AND metadata-identical. A daemon + the base image are present on this
+      # runner, so the -m live variant executes here (Linux-authoritative).
+      - name: Run docker host-unchanged live detector
+        run: .venv/bin/pytest tests/test_docker_host_unchanged.py -m live -p no:cacheprovider
+
       - name: Build BYOD template Docker image
         run: docker build -t byod-custom-image:0.1.0 templates/byod_smoke_test/
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install format check typecheck test test-live test-smoke verify verify-noextra clean run lint docs-indexes docker-image docker-image-full coder-eval-runtime docker-images
+.PHONY: help install format check typecheck test test-live test-smoke verify verify-noextra clean run lint docs-indexes docker-image docker-image-full coder-eval-runtime docker-images test-docker-detectors
 
 # Single source of the installed coder-eval version (used to tag the docker
 # images). Referenced lazily inside the docker recipes, so it doesn't run on
@@ -100,6 +100,10 @@ coder-eval-runtime:  ## Build the relocatable runtime kit image (COPY --from sou
 docker-images: docker-image coder-eval-runtime  ## Build BOTH base images (agent for rebase + runtime kit for inject); no creds
 	@echo "Built coder-eval-agent + coder-eval-runtime — ready for both rebase and inject tasks."
 
+test-docker-detectors:  ## Run the docker isolation detectors (host-unchanged proxy + criteria absence + baked-image scan). Daemon-less; CI-cheap.
+	uv run pytest tests/test_docker_host_unchanged.py tests/test_docker_criteria_isolation.py \
+		tests/test_docker_image_no_answer_leak.py -m "not live"
+
 docker-image-full:  ## Build with the UiPath extra (opt-in; uipath resolves from public PyPI, no credentials needed). Codex is always baked in.
 	@VERSION=$$($(VERSION_CMD)); \
 	echo "Building coder-eval-agent:$$VERSION (full: + uipath extra)"; \

docs/TASK_DEFINITION_GUIDE.md

@@ -1274,11 +1274,35 @@ pre_run:
 | `command` | *required* | Shell command to execute (supports pipes, redirects, `&` for background) |
 | `timeout` | 30 | Maximum seconds to wait (1–300) |
 | `fail_on_error` | `true` | When true, failure aborts evaluation with `FinalStatus.ERROR` |
+| `runs_in` | `host` | Under `driver: docker`, where the command runs: `host` or `agent` (see below) |
 
 Commands run sequentially with `cwd` set to the sandbox directory. stdout and stderr are
 captured in `pre_run_results` on the evaluation result (truncated to 100KB each). When a
 command fails with `fail_on_error: true`, remaining commands are skipped.
 
+**`runs_in: host | agent` (docker only).** Under `driver: docker` the container runs the
+agent turn only; the grading material and helper scripts live host-side. So each `pre_run`
+command's `runs_in` decides *where* it runs:
+
+- `runs_in: host` (default) — runs on the **host**, before the container, into a staging dir
+  whose contents seed the agent workspace. The right place for setups that need the host repo
+  or credentials (fixture copies, seed scripts). This preserves behavior for the vast majority
+  of tasks — a `pre_run` with no `runs_in` runs host-side exactly as before.
+- `runs_in: agent` — runs **inside** the coder_eval container, in the seeded workspace, before
+  the agent turn. Use it for setups that must run where they will be used — e.g. `uv sync`
+  building a virtualenv the agent runs against, or `uip codedagent setup` (live-tenant
+  provisioning). It is SDK-agnostic (executed by the in-container orchestrator, not a per-SDK
+  hook) and **self-contained by contract**: it gets the image + seeded workspace + forwarded
+  creds + network, but **not** the graders/criteria/skills `tests/` tree — so it cannot see
+  grading material.
+
+`post_run` has **no** `runs_in` — under docker it is always host-only (runs after the container
+exits, over the copied-out workspace). Under `driver: tempdir` there is no container, so
+`runs_in` is a no-op: `agent` behaves identically to `host` (everything runs in the one
+sandbox). A docker `pre_run` that needs the container (`uv sync` / `uip codedagent setup`) but
+is left at the `host` default is rejected at task resolution with an instruction to mark it
+`runs_in: agent`.
+
 **Execution order:**
 
 1. Sandbox setup (template sources applied, venv/node packages installed)

src/coder_eval/cli/plan_command.py

@@ -59,6 +59,7 @@ def plan_command(
     check_api_keys()
 
     # Lazy import to avoid circular dependency at module level
+    from ..orchestration.docker_guard import DockerPreRunHostUnsafeError, validate_docker_pre_run_host_safety
     from ..orchestration.early_stop import EarlyStopConfigError, validate_early_stop
     from ..orchestration.experiment import DEFAULT_EXPERIMENT_PATH, load_experiment, resolve_task_for_variant
 
@@ -136,6 +137,8 @@ def plan_command(
                     resolved, _lineage, _ = resolve_task_for_variant(default_exp, task, exp_def, variant)
                     # Early-stop guardrails (no-op unless a criterion carries a stop_early: block).
                     validate_early_stop(resolved)
+                    # Interim docker guard: docker pre_run that must run in-container.
+                    validate_docker_pre_run_host_safety(resolved)
                     agent_type = str(resolved.agent.type) if resolved.agent else "unknown"
                     agent_model = resolved.agent.model if resolved.agent else None
                     model_str = f" ({agent_model})" if agent_model else ""
@@ -145,6 +148,11 @@ def plan_command(
                     # failures, which stay soft): flip the plan exit code.
                     console.print(f"    [red]Variant '{variant.variant_id}': early-stop config error - {e}[/red]")
                     all_valid = False
+                except DockerPreRunHostUnsafeError as e:
+                    # Interim docker guard: same hard-error treatment (flip the
+                    # exit code) — a docker pre_run that must run in-container.
+                    console.print(f"    [red]Variant '{variant.variant_id}': docker config error - {e}[/red]")
+                    all_valid = False
                 except Exception as e:
                     console.print(f"    [red]Variant '{variant.variant_id}': resolution failed - {e}[/red]")
 

src/coder_eval/cli/run_task_internal_command.py

@@ -158,6 +158,21 @@ def _watch_host_heartbeat() -> None:
     # Absent -> None -> standard run_dir/artifacts workspace.
     workspace_dir_raw = context.get("workspace_dir")
     workspace_dir = Path(workspace_dir_raw) if workspace_dir_raw else None
+    # HARNESS-OUTSIDE: under docker, post_run is HOST-only and pre_run is SPLIT by
+    # each command's `runs_in`. The host forwards exactly one of two knobs:
+    #   - skip_pre_post_commands=True  → no `runs_in: agent` pre_run; the container
+    #     runs the agent turn only (both phases suppressed).
+    #   - pre_run_in_container=True    → ≥1 `runs_in: agent` pre_run; the container
+    #     runs ONLY that `agent` subset (in the seeded workspace, before the agent)
+    #     and still skips all post_run (post is host-only).
+    # Both absent -> False (in-process driver), so pre/post run in-process as before.
+    skip_pre_post_commands: bool = bool(context.get("skip_pre_post_commands", False))
+    pre_run_in_container: bool = bool(context.get("pre_run_in_container", False))
+    # Host-produced workspace-seed mount: the in-container orchestrator copies
+    # its contents into the sandbox after template materialization, before the
+    # agent starts (seed wins over template starters). Absent -> None -> no-op.
+    workspace_seed_dir_raw = context.get("workspace_seed_dir")
+    workspace_seed_dir = Path(workspace_seed_dir_raw) if workspace_seed_dir_raw else None
     config_lineage = {k: ConfigLineageEntry.model_validate(v) for k, v in (context.get("config_lineage") or {}).items()}
     # Prefer the host's raw source_yaml so task.json's audit trail matches
     # the in-process driver. Fall back to the staged (post-override) YAML
@@ -169,7 +184,11 @@ def _watch_host_heartbeat() -> None:
     # Orchestrator's `task_file.parent` reasoning -- specifically the
     # `TASK_DIR` env exposed to `run_command` criteria -- resolves to the
     # original host task directory rather than `/work/input/`.
-    task, source_yaml = load_task(task_yaml)
+    # allow_empty_criteria=True: the staged task.yaml is agent_safe_dump-stripped
+    # (success_criteria: []) so the agent container carries no grading material;
+    # the host holds the real criteria and grades after the container exits. This
+    # is the ONLY caller allowed to bypass the authored-empty-criteria guard.
+    task, source_yaml = load_task(task_yaml, allow_empty_criteria=True)
     if host_source_yaml is not None:
         source_yaml = host_source_yaml
     # The path below is never re-read; it only seeds Orchestrator's TASK_DIR.
@@ -197,6 +216,9 @@ def _watch_host_heartbeat() -> None:
         config_lineage=config_lineage,
         replicate_index=replicate_index,
         workspace_dir=workspace_dir,
+        workspace_seed_dir=workspace_seed_dir,
+        skip_pre_post_commands=skip_pre_post_commands,
+        pre_run_in_container=pre_run_in_container,
     )
 
     # Install the stdout-NDJSON stream callback so per-tool-call events

src/coder_eval/evaluation/host_commands.py

@@ -0,0 +1,174 @@
+"""Shared execution of ``pre_run``/``post_run`` shell command lists.
+
+The core loop is a free function :func:`run_command_list` so it can run both
+in-process (via ``Orchestrator._run_command_list``, which delegates here) AND
+host-side over a copied-out workspace (under ``--driver docker``, where the
+graders/helper scripts live only on the host and post-run teardown must run
+after the container exits). Keeping the loop in one place preserves the exact
+semantics — ``PreRunCommand.fail_on_error`` abort, ``PostRunCommand``
+informational/non-fatal, per-command timeout, output truncation, line-by-line
+streaming to a logger, and a caller-supplied ``cwd`` — regardless of caller.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from ..models import PostRunCommand, PostRunResult, PreRunCommand
+
+
+logger = logging.getLogger("coder_eval.orchestrator")
+
+# Truncate captured stdout/stderr to 100KB per stream.
+DEFAULT_MAX_OUTPUT = 100_000
+# StreamReader per-line buffer (256KB).
+DEFAULT_STREAM_LIMIT = 262_144
+
+
+async def _pump_stream(
+    stream: asyncio.StreamReader | None,
+    log_fn: Callable[..., None],
+    label: str,
+    chunks: list[str],
+) -> None:
+    """Read ``stream`` line-by-line, log each non-empty line via ``log_fn``,
+    and accumulate the raw text into ``chunks`` for later capture.
+
+    Forwards subprocess output to the logger in real time while preserving it
+    for ``PostRunResult``. If a single line exceeds the StreamReader buffer
+    (rare — only for binary-ish or malformed output), it is drained as a chunk
+    and logged as a partial.
+    """
+    if stream is None:
+        return
+    while True:
+        try:
+            raw = await stream.readline()
+        except asyncio.LimitOverrunError as e:
+            # Single line larger than the buffer; drain the buffered bytes so
+            # readline() can make progress on the next iteration.
+            raw = await stream.readexactly(e.consumed)
+            text = raw.decode(errors="replace")
+            chunks.append(text)
+            log_fn("[%s] (partial line, %d bytes)", label, len(raw))
+            continue
+        if not raw:
+            break
+        text = raw.decode(errors="replace")
+        chunks.append(text)
+        line = text.rstrip()
+        if line:
+            log_fn("[%s] %s", label, line)
+
+
+async def run_command_list(
+    commands: list[PreRunCommand] | list[PostRunCommand],
+    results: list[PostRunResult],
+    label: str,
+    *,
+    cwd: Path | str,
+    max_output: int = DEFAULT_MAX_OUTPUT,
+    stream_limit: int = DEFAULT_STREAM_LIMIT,
+) -> None:
+    """Run a list of shell commands with ``cwd``, capturing output.
+
+    stdout/stderr are streamed line-by-line to the orchestrator logger and
+    accumulated into ``results`` for the report (truncated to ``max_output``
+    per stream). ``label`` is used in stream/log labels (e.g. ``"pre_run"`` ->
+    ``[pre_run stdout]``).
+
+    For commands carrying ``fail_on_error=True`` (PreRunCommand only), a
+    non-zero exit, timeout, or exception appends the failure result and then
+    raises ``RuntimeError``, aborting the loop. PostRunCommand never has
+    ``fail_on_error`` set, so failures are warning-logged and the loop
+    continues — preserving existing post-run "informational only" semantics.
+    """
+    if not commands:
+        return
+
+    cwd_str = str(cwd)
+    human = label.replace("_", "-").capitalize()  # "pre_run" -> "Pre-run"
+
+    for cmd in commands:
+        fail_on_error = isinstance(cmd, PreRunCommand) and cmd.fail_on_error
+        start = time.time()
+        logger.info("Running %s command: %s", human.lower(), cmd.command)
+
+        try:
+            proc = await asyncio.create_subprocess_shell(
+                cmd.command,
+                cwd=cwd_str,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                limit=stream_limit,
+            )  # nosec B602,B604 - commands come from task YAML, not user input
+
+            stdout_chunks: list[str] = []
+            stderr_chunks: list[str] = []
+
+            try:
+                await asyncio.wait_for(
+                    asyncio.gather(
+                        _pump_stream(proc.stdout, logger.info, f"{label} stdout", stdout_chunks),
+                        _pump_stream(proc.stderr, logger.warning, f"{label} stderr", stderr_chunks),
+                        proc.wait(),
+                    ),
+                    timeout=cmd.timeout,
+                )
+            except TimeoutError:
+                proc.kill()
+                await proc.wait()
+                results.append(
+                    PostRunResult(
+                        command=cmd.command,
+                        stdout="".join(stdout_chunks)[:max_output],
+                        stderr="".join(stderr_chunks)[:max_output],
+                        error=f"Timed out after {cmd.timeout}s",
+                        duration_seconds=time.time() - start,
+                    )
+                )
+                if fail_on_error:
+                    raise RuntimeError(f"{human} command timed out after {cmd.timeout}s: {cmd.command!r}") from None
+                logger.warning("%s command '%s' timed out after %ds", human, cmd.command, cmd.timeout)
+                continue
+
+            stdout_text = "".join(stdout_chunks)[:max_output]
+            stderr_text = "".join(stderr_chunks)[:max_output]
+            results.append(
+                PostRunResult(
+                    command=cmd.command,
+                    exit_code=proc.returncode,
+                    stdout=stdout_text,
+                    stderr=stderr_text,
+                    duration_seconds=time.time() - start,
+                )
+            )
+            if proc.returncode != 0:
+                if fail_on_error:
+                    raise RuntimeError(f"{human} command failed (exit {proc.returncode}): {cmd.command!r}")
+                logger.warning(
+                    "%s command '%s' exited with code %d: %s",
+                    human,
+                    cmd.command,
+                    proc.returncode,
+                    stderr_text[:200],
+                )
+        except RuntimeError:
+            # Propagate abort signal from fail_on_error=True branches unchanged;
+            # otherwise the catch-all below would re-wrap it as a new RuntimeError.
+            raise
+        except Exception as e:
+            results.append(
+                PostRunResult(
+                    command=cmd.command,
+                    error=str(e),
+                    duration_seconds=time.time() - start,
+                )
+            )
+            if fail_on_error:
+                raise RuntimeError(f"{human} command failed: {cmd.command!r}") from e
+            logger.warning("%s command '%s' failed: %s", human, cmd.command, e)