From 6a862cb1c3934f51f6ad0976f849ec8e73ca83c5 Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Thu, 29 Oct 2026 11:53:31 +0200 Subject: [PATCH 1/8] Put the agent behind a harness contract, with Claude Code as the first The host engine assumed one agent. Its command line, environment, skills directory and "is it installed" check were all written for the `claude` CLI inside the solver, so adding another agent meant copying the solver. Add `skillscope.harness`: a small `Harness` contract (binary, skills_dir, command, env, normalize, require), a registry, and `ClaudeCode`, which is the command line and environment the engine already used, moved out unchanged. `normalize` translates a harness's own events into the stream-json subset the engines already read, so stop rules, budgets, scorers and the judge are untouched and written once for every harness. `no_sandbox.harness_solver(harness, ...)` is the old solver, generalised. `claude_code_no_sandbox` keeps its signature and calls it with Claude Code, so every caller behaves as before. `_stream_until` gains an optional `normalize` so the stop rule sees canonical events only, and `install_skill` takes the harness's skills directory. Nothing user-visible changes: no new flag, and the 505 existing tests pass unmodified, including the one that drives the real solver against a stub `claude`. Choosing a harness from the command line, and moving the preflight, credential check, spend flags, CI model pin and judge behind the contract, are left for the change that adds a second harness; docs/harnesses.md lists them. New tests: the registry; a golden check that the Claude Code command and environment equal what the engine built inline; a contract test that runs once per registered harness against recorded output in tests/fixtures/harness and fails when a harness ships none; and a harness whose events look nothing like Claude Code's, driven both through `_stream_until` and through the solver under inspect. Each was checked by breaking the code it covers and seeing it fail. Signed-off-by: Eugene Volen --- README.md | 2 + docs/harnesses.md | 93 ++++ docs/usage.md | 3 +- pyproject.toml | 2 +- skillscope/engine/no_sandbox.py | 114 +++-- skillscope/harness/__init__.py | 46 ++ skillscope/harness/base.py | 92 ++++ skillscope/harness/claude_code.py | 44 ++ .../harness/claude-code/expected.json | 6 + .../harness/claude-code/no_trigger.jsonl | 3 + .../harness/claude-code/trigger.jsonl | 4 + tests/test_harness.py | 447 ++++++++++++++++++ 12 files changed, 818 insertions(+), 38 deletions(-) create mode 100644 docs/harnesses.md create mode 100644 skillscope/harness/__init__.py create mode 100644 skillscope/harness/base.py create mode 100644 skillscope/harness/claude_code.py create mode 100644 tests/fixtures/harness/claude-code/expected.json create mode 100644 tests/fixtures/harness/claude-code/no_trigger.jsonl create mode 100644 tests/fixtures/harness/claude-code/trigger.jsonl create mode 100644 tests/test_harness.py diff --git a/README.md b/README.md index f3600f0..5db84d9 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ the repo under test — worth adding to `.gitignore`. * [docs/authoring-evals.md](docs/authoring-evals.md): writing a skill's tests. * [docs/usage.md](docs/usage.md): every flag, every workflow input, and what each check actually asserts. +* [docs/harnesses.md](docs/harnesses.md): what an agent harness is, and how to + add one. ## Development diff --git a/docs/harnesses.md b/docs/harnesses.md new file mode 100644 index 0000000..59dd215 --- /dev/null +++ b/docs/harnesses.md @@ -0,0 +1,93 @@ + + +# Harnesses + +A **harness** is the agent that is handed a prompt and a workspace and does the +work: Claude Code, GitHub Copilot CLI, Codex, and so on. It is a different +question from where the agent runs. `--engine` says *where* (on the host, or in +a container); the harness says *what*. The engines drive a harness as a +subprocess, so the contract between them is small, and this page is that +contract. + +Today one harness is registered, `claude-code`. The contract exists so that a +second one is an adapter and a set of tests rather than a change to the engines. + +## What a harness provides + +A harness is a subclass of `skillscope.harness.Harness`, registered in +`skillscope/harness/__init__.py`. + +| Member | What it is | +| --- | --- | +| `name` | The name reports and `--harness` use. | +| `binary` | The executable looked up on `PATH`. | +| `skills_dir` | Where, relative to the workspace, the harness discovers skills (`.claude/skills` for Claude Code). The engine stages each skill there. | +| `has_skill_tool` | Whether activating a skill is a tool call in the stream. A harness that activates a skill by reading its `SKILL.md` sets this `False`, and routing reads activation from the file access instead. | +| `prompt_on_stdin` | Whether the prompt is written to stdin. A harness that takes it as an argument builds it into `command` and sets this `False`; stdin is closed either way, because some CLIs wait on it. | +| `require()` | Refuse to start when the harness is not installed. The default checks `binary` is on `PATH`. | +| `command(spec)` | The argv that runs one case, headless, streaming JSON events. | +| `env(spec)` | The environment of that process, including whatever points it at `spec.config_dir` so the runner's own settings and skills stay out of the session. | +| `normalize(event)` | Turns one event the harness printed into canonical events (below). | + +`spec` is a `RunSpec`: the workspace, the prompt, the model and effort, an +optional throwaway config directory, and any spend or session flags the caller +built. + +## The canonical event form + +The engines read one event shape, and a harness translates into it. It is +Claude Code's stream-json, reduced to what is read: + +| Event | Carries | +| --- | --- | +| `{"type": "system", "subtype": "init", "skills": [...]}` | The skills the session registered. Optional: a harness that cannot say reports nothing, and the room check says it did not run rather than reporting an empty room. | +| `{"type": "assistant", "message": {"content": [{"type": "tool_use", "name": ..., "input": {...}}]}}` | A tool call. Activating a skill is a `tool_use` named `Skill` with `input.skill` set. | +| `{"type": "user", "message": {"content": [{"type": "tool_result", "content": ...}]}}` | A tool result. | +| `{"type": "result", "result": "..."}` | The end of the run, and its closing answer. A stream without one is a run that did not finish. | + +A harness whose stream already has this shape keeps the default `normalize`, +which passes events through. Any other maps its events onto these and returns an +empty list for events that carry nothing the engines read. Every stop rule, +budget, score and judge sees canonical events only, never the raw ones, so each +is written once for every harness. + +## Adding a harness + +1. Subclass `Harness` in `skillscope/harness/.py`, fill in the members + above, and register it in `skillscope/harness/__init__.py`. +2. Record two real runs of the CLI, headless, on a trivial skill: one where it + activates the skill and finishes, one where it finishes without activating + one. Scrub tokens and paths. Put them in + `tests/fixtures/harness//trigger.jsonl` and `no_trigger.jsonl`, with an + `expected.json` saying what the engine should read out of each (see the + Claude Code one) and the CLI version they were recorded with. +3. Run `python -m unittest tests.test_harness`. The contract test runs once per + registered harness and fails if there are no fixtures. It checks activation, + completion, the closing answer, that the command carries the model and effort + and no credential, that the config directory is isolated, and that a skill + is staged where the harness looks and without its `evals/` directory. +4. Say in the pull request how the harness differs from the others in ways a + reader of a report should know: how it activates skills, what it cannot + report, and what it needs to authenticate. + +A report from one harness is its own result. Harnesses send different system +prompts, offer different tools, and activate skills in different ways, so a score +from one is not comparable with a score from another. + +## What is not yet behind the contract + +The contract covers staging, the command, the environment and the events. These +still assume Claude Code, and a second harness will need to move them behind it: + +* choosing a harness from the command line (there is no `--harness` flag yet); +* the reachability preflight and the credential check that decides whether the + runner's own config can be kept out of a routing run (`ANTHROPIC_*`); +* the spend-limit flags, which are probed from `claude --help`; +* the CI model pin, which forces `opus`; +* the judge, which uses an Anthropic model through `inspect_ai`; +* `.claude/skills` as the path routing treats as a skills inspection, rather than + a harness's own `skills_dir`. diff --git a/docs/usage.md b/docs/usage.md index 442d1b1..3ad6a34 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -259,7 +259,8 @@ changes. | `claude-code` | The real CLI, via `inspect_swe` | a sandbox | `skillscope[verify]`, Linux only | Both drive the agent a skill is written for, so what differs between them -is **where the agent runs**, not what it is. That is the axis worth choosing +is **where the agent runs**, not what it is. What it is -- the harness -- is a +separate question, answered in [harnesses.md](harnesses.md). That is the axis worth choosing along: the host measures the machine as it is, with whatever else is installed on it, and the sandbox measures the skill alone. When the two disagree, the disagreement is usually a fact about one of those environments rather than diff --git a/pyproject.toml b/pyproject.toml index 3ab8e4e..3b13eb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ verify = ["skillscope[inspect]", "inspect-swe>=0.2.71"] skillscope = "skillscope.cli:main" [tool.setuptools] -packages = ["skillscope", "skillscope.engine"] +packages = ["skillscope", "skillscope.engine", "skillscope.harness"] [tool.setuptools.package-data] skillscope = ["data/*.json", "schema/*.json"] diff --git a/skillscope/engine/no_sandbox.py b/skillscope/engine/no_sandbox.py index 8d31bae..e41b8d6 100644 --- a/skillscope/engine/no_sandbox.py +++ b/skillscope/engine/no_sandbox.py @@ -33,6 +33,7 @@ from typing import Callable from .. import agent as legacy_agent +from .. import harness as harnesses from .. import usage from .. import routing as routing_core @@ -64,7 +65,7 @@ EVAL_FIXTURE_DIRNAME = "evals" -def require_local() -> None: +def require_local(harness: harnesses.Harness | None = None) -> None: """This driver runs on the host, so settle the sandbox on the host. Two cases, and they are not the same question. @@ -96,8 +97,7 @@ def require_local() -> None: "Unset it to run on the host, or use --engine claude-code for a " "sandboxed run of the real harness on Linux." ) - if not shutil.which("claude"): - raise SystemExit("error: 'claude' CLI not found on PATH") + (harness or harnesses.get()).require() async def _workspace() -> str: @@ -251,9 +251,15 @@ async def _stream_until( workspace: str, env: dict, stop_when: Callable[[dict], str | None], + normalize: Callable[[dict], list[dict]] | None = None, ) -> tuple[list[dict], str | None, int | None, str]: """Run the CLI, reading its stream, and stop the moment `stop_when` says to. + `normalize` turns each event the CLI prints into the canonical events the + rest of the engine reads (see `harness.Harness.normalize`). The stop rule + sees the canonical events, never the raw ones, so one rule serves every + harness. Left `None`, events pass through as they came. + The reason this exists rather than `inspect_ai.util.subprocess`: that call returns once the process has exited, so nothing it gives back can arrive in time to end the run early. For a behavioral case that is exactly right -- @@ -323,8 +329,11 @@ async def _drain_stderr() -> None: event = json.loads(line) except json.JSONDecodeError: continue - events.append(event) - stop_reason = stop_when(event) + for canonical in normalize(event) if normalize else [event]: + events.append(canonical) + stop_reason = stop_when(canonical) + if stop_reason is not None: + break if stop_reason is not None: break finally: @@ -338,15 +347,18 @@ async def _drain_stderr() -> None: return events, stop_reason, proc.returncode, "".join(errors) -def install_skill(skill_dir: Path, workspace: str) -> None: +def install_skill( + skill_dir: Path, workspace: str, skills_dir: str | None = None +) -> None: """Put the skill where the real harness looks for it. + `/` inside the workspace, which for Claude Code is `.claude/skills/` inside a directory the CLI is given with - `--add-dir`, which is what `inspect_swe` does via its own `skills=` - argument. Staging is the driver's job, and this driver is the one that has - to do it by hand: skip it and the agent runs with no skill at all, - answering from the prompt and scoring like it -- which looks like a bad - skill rather than a missing one. + `--add-dir`, and what `inspect_swe` does via its own `skills=` argument. + Another harness names its own directory. Staging is the driver's job, and + this driver is the one that has to do it by hand: skip it and the agent + runs with no skill at all, answering from the prompt and scoring like it -- + which looks like a bad skill rather than a missing one. Everything except the skill's own test suite. This used to copy the directory wholesale, which put `evals/evals.json` inside the room the agent @@ -364,7 +376,7 @@ def install_skill(skill_dir: Path, workspace: str) -> None: are read from the skill directory on disk and seeded into the *workspace*, which is a different path. """ - dest = Path(workspace) / ".claude" / "skills" / skill_dir.name + dest = Path(workspace) / (skills_dir or harnesses.get().skills_dir) / skill_dir.name dest.parent.mkdir(parents=True, exist_ok=True) shutil.copytree( skill_dir, @@ -396,7 +408,8 @@ def _record_room(state, event: dict, room_names: list[str] | None) -> None: state.store.set(EXTRA_SKILLS_KEY, routing_core.init_extra_skills(event, room_names)) -def claude_code_no_sandbox( +def harness_solver( + harness: harnesses.Harness, model: str | None, effort: str | None, skill_dir: Path, @@ -407,6 +420,13 @@ def claude_code_no_sandbox( ): """Solver: install the skill, run the real CLI once, record what it did. + `harness` is the agent that runs the case. It decides where the skill is + staged, the command line and environment of the process, and how its output + is translated into the canonical events everything below reads; this driver + owns the rest, and owns it once for every harness. `claude_code_no_sandbox` + is this with Claude Code, which is what every caller used before there was + a choice. + `extra_flags` are the CLI's own controls -- `--max-budget-usd` and `--no-session-persistence` -- built by `routing.host_cost_flags`, which probes `--help` first: an older build rejects an unknown flag and every @@ -444,34 +464,32 @@ def claude_code_no_sandbox( from inspect_ai.solver import solver @solver - def _claude_code_no_sandbox(): + def _harness_solver(): async def solve(state, generate): from inspect_ai.util import subprocess as sandbox_subprocess workspace = await _workspace() - install_skill(skill_dir, workspace) + install_skill(skill_dir, workspace, harness.skills_dir) prompt = state.input_text - cmd = [ - shutil.which("claude"), "-p", - "--output-format", "stream-json", "--verbose", - "--dangerously-skip-permissions", - "--add-dir", workspace, - ] - if model: - cmd += ["--model", model] - if effort: - cmd += ["--effort", effort] - cmd += list(extra_flags or []) - - env = legacy_agent.claude_env() - if config_dir is not None: - env["CLAUDE_CONFIG_DIR"] = str(config_dir) + spec = harnesses.RunSpec( + workspace=workspace, + prompt=prompt, + model=model, + effort=effort, + config_dir=config_dir, + extra_flags=tuple(extra_flags or ()), + ) + cmd = harness.command(spec) + env = harness.env(spec) + # Closed either way: a CLI that is not given its prompt on stdin + # can still sit waiting on it. + stdin_text = prompt if harness.prompt_on_stdin else "" events: list[dict] = [] if stop_when_factory is None: result = await sandbox_subprocess( - cmd, input=prompt, cwd=workspace, env=env, + cmd, input=stdin_text, cwd=workspace, env=env, ) returncode, stderr = result.returncode, result.stderr or "" for line in (result.stdout or "").splitlines(): @@ -479,13 +497,14 @@ async def solve(state, generate): if not line: continue try: - events.append(json.loads(line)) + events.extend(harness.normalize(json.loads(line))) except json.JSONDecodeError: continue else: async with _subprocess_concurrency(): events, stop_reason, returncode, stderr = await _stream_until( - cmd, prompt, workspace, env, stop_when_factory() + cmd, stdin_text, workspace, env, stop_when_factory(), + harness.normalize, ) # Recorded whatever it was, including `None` for a stream that # ended on its own. A stop this driver performed leaves no @@ -495,7 +514,7 @@ async def solve(state, generate): if not events: raise RuntimeError( - "claude produced no parseable stream-json output " + f"{harness.binary} produced no parseable stream-json output " f"(exit {returncode}). stderr: {stderr[:300]}" ) @@ -506,10 +525,33 @@ async def solve(state, generate): messages, final = events_to_messages(events, prompt) state.messages.extend(messages) state.output = ModelOutput.from_content( - model=model or "claude", content=final or "(no final message)" + model=model or harness.binary, + content=final or "(no final message)", ) return state return solve - return _claude_code_no_sandbox() + return _harness_solver() + + +def claude_code_no_sandbox( + model: str | None, + effort: str | None, + skill_dir: Path, + config_dir: Path | None = None, + extra_flags: list[str] | None = None, + stop_when_factory: Callable[[], Callable[[dict], str | None]] | None = None, + room_names: list[str] | None = None, +): + """`harness_solver` with Claude Code, the harness every caller has used.""" + return harness_solver( + harnesses.get("claude-code"), + model, + effort, + skill_dir, + config_dir, + extra_flags, + stop_when_factory, + room_names, + ) diff --git a/skillscope/harness/__init__.py b/skillscope/harness/__init__.py new file mode 100644 index 0000000..c641d5d --- /dev/null +++ b/skillscope/harness/__init__.py @@ -0,0 +1,46 @@ +# Copyright Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: MIT + +"""The agent harnesses skillscope can drive. + +A harness is *what* runs a case; the engine is *where*. See `base.Harness` for +the contract and `docs/harnesses.md` for how to add one. +""" + +from __future__ import annotations + +from .base import Harness, RunSpec +from .claude_code import ClaudeCode + +DEFAULT = "claude-code" + +_REGISTRY: dict[str, Harness] = {} + + +def register(harness: Harness) -> Harness: + """Make `harness` selectable by its name. Registering a name twice is a bug.""" + if harness.name in _REGISTRY: + raise ValueError(f"harness {harness.name!r} is already registered") + _REGISTRY[harness.name] = harness + return harness + + +def names() -> list[str]: + return sorted(_REGISTRY) + + +def get(name: str | None = None) -> Harness: + """The harness called `name`, or the default one when none is named.""" + key = name or DEFAULT + try: + return _REGISTRY[key] + except KeyError: + raise SystemExit( + f"error: unknown harness {key!r}. Known harnesses: {', '.join(names())}." + ) from None + + +register(ClaudeCode()) + +__all__ = ["DEFAULT", "Harness", "RunSpec", "get", "names", "register"] diff --git a/skillscope/harness/base.py b/skillscope/harness/base.py new file mode 100644 index 0000000..32c25aa --- /dev/null +++ b/skillscope/harness/base.py @@ -0,0 +1,92 @@ +# Copyright Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: MIT + +"""What a harness is, as far as the engines are concerned. + +A *harness* is the agent that is handed a prompt and a workspace and does the +work: Claude Code, GitHub Copilot CLI, Codex, and so on. It is a different +question from where it runs (`--engine`: on the host or in a container), and +this module is the contract for answering it. + +The engines drive a harness as a subprocess, so the contract is small: how to +find it, where it looks for skills, how to start one case, what environment to +start it in, and how to read what it prints. Everything else -- staging, the +stop rule, usage, grading -- stays in the engine, once, for every harness. + +Adding a harness is one subclass of `Harness`, registered in +`skillscope/harness/__init__.py`, plus recorded output for the contract test in +`tests/test_harness.py`. `docs/harnesses.md` has the checklist. +""" + +from __future__ import annotations + +import shutil +from abc import ABC, abstractmethod +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class RunSpec: + """Everything one case hands to a harness. + + A value object rather than keyword arguments so a harness that needs one + more input in future (a trust flag, a home directory) reads it from here + instead of every caller growing a parameter. + """ + + workspace: str + prompt: str + model: str | None = None + effort: str | None = None + # A throwaway config directory, so the runner's own settings and user-level + # skills stay out of the session. `None` when the caller does not isolate. + config_dir: Path | None = None + # The harness's own spend or session controls, already built by the caller. + extra_flags: tuple[str, ...] = () + + +class Harness(ABC): + """One agent harness. Subclass, fill in the class attributes, register.""" + + # The name `--harness` and the reports use. + name: str + # The executable looked up on PATH. + binary: str + # Where, relative to the workspace, the harness discovers skills. + skills_dir: str + # Whether activating a skill is a tool call the stream names. A harness with + # no such call (it activates a skill by reading its SKILL.md) sets this + # `False`, and activation is read from the file access instead. + has_skill_tool: bool = True + # Whether the prompt is written to the process's stdin. A harness that takes + # it as an argument builds that into `command` and sets this `False`; its + # stdin is closed either way, because some CLIs wait on it. + prompt_on_stdin: bool = True + + def require(self) -> None: + """Refuse to start when the harness is not installed.""" + if not shutil.which(self.binary): + raise SystemExit(f"error: '{self.binary}' CLI not found on PATH") + + @abstractmethod + def command(self, spec: RunSpec) -> list[str]: + """The argv that runs one case, headless, streaming JSON events.""" + + @abstractmethod + def env(self, spec: RunSpec) -> dict[str, str]: + """The environment that process gets, config isolation included.""" + + def normalize(self, event: dict) -> list[dict]: + """Translate one event the harness printed into canonical events. + + The canonical form is the one the engines already read: Claude Code's + stream-json, reduced to `assistant` messages carrying `tool_use` + blocks, `user` messages carrying `tool_result` blocks, the + `system`/`init` event naming the skills the session registered, and a + final `result` event. A harness whose stream already has that shape + keeps this default; any other maps its events onto it, and returns an + empty list for the ones that carry nothing the engines read. + """ + return [event] diff --git a/skillscope/harness/claude_code.py b/skillscope/harness/claude_code.py new file mode 100644 index 0000000..edc659d --- /dev/null +++ b/skillscope/harness/claude_code.py @@ -0,0 +1,44 @@ +# Copyright Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: MIT + +"""Claude Code as a harness. + +Not new behaviour: this is the command line and environment the host engine +has always used for the `claude` CLI, moved behind the `Harness` contract so +the engine no longer assumes it. Claude Code's stream-json is the canonical +event form, so `normalize` is the default identity. +""" + +from __future__ import annotations + +import shutil + +from .. import agent +from .base import Harness, RunSpec + + +class ClaudeCode(Harness): + name = "claude-code" + binary = "claude" + skills_dir = ".claude/skills" + + def command(self, spec: RunSpec) -> list[str]: + cmd = [ + shutil.which(self.binary), "-p", + "--output-format", "stream-json", "--verbose", + "--dangerously-skip-permissions", + "--add-dir", spec.workspace, + ] + if spec.model: + cmd += ["--model", spec.model] + if spec.effort: + cmd += ["--effort", spec.effort] + cmd += list(spec.extra_flags) + return cmd + + def env(self, spec: RunSpec) -> dict[str, str]: + env = agent.claude_env() + if spec.config_dir is not None: + env["CLAUDE_CONFIG_DIR"] = str(spec.config_dir) + return env diff --git a/tests/fixtures/harness/claude-code/expected.json b/tests/fixtures/harness/claude-code/expected.json new file mode 100644 index 0000000..44b0a3a --- /dev/null +++ b/tests/fixtures/harness/claude-code/expected.json @@ -0,0 +1,6 @@ +{ + "note": "Synthetic: hand-written to the stream-json shape, not a recording.", + "skills": ["alpha"], + "trigger": {"activation": "alpha", "completed": true, "final": "done"}, + "no_trigger": {"activation": null, "completed": true, "final": "That is not something a skill is needed for."} +} diff --git a/tests/fixtures/harness/claude-code/no_trigger.jsonl b/tests/fixtures/harness/claude-code/no_trigger.jsonl new file mode 100644 index 0000000..2f055f2 --- /dev/null +++ b/tests/fixtures/harness/claude-code/no_trigger.jsonl @@ -0,0 +1,3 @@ +{"type":"system","subtype":"init","skills":["alpha"]} +{"type":"assistant","message":{"content":[{"type":"text","text":"That is not something a skill is needed for."}]}} +{"type":"result","subtype":"success","result":"That is not something a skill is needed for."} diff --git a/tests/fixtures/harness/claude-code/trigger.jsonl b/tests/fixtures/harness/claude-code/trigger.jsonl new file mode 100644 index 0000000..6770bd2 --- /dev/null +++ b/tests/fixtures/harness/claude-code/trigger.jsonl @@ -0,0 +1,4 @@ +{"type":"system","subtype":"init","skills":["alpha","a-stray-skill"]} +{"type":"assistant","message":{"usage":{"input_tokens":11,"output_tokens":7},"content":[{"type":"tool_use","id":"t1","name":"Skill","input":{"skill":"alpha"}}]}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","content":"Launching skill: alpha"}]}} +{"type":"result","subtype":"success","result":"done"} diff --git a/tests/test_harness.py b/tests/test_harness.py new file mode 100644 index 0000000..56188e8 --- /dev/null +++ b/tests/test_harness.py @@ -0,0 +1,447 @@ +# Copyright Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: MIT + +"""The harness contract: what every agent harness has to provide. + +Three kinds of test, in the order a new harness meets them: + +* the registry, which is how `--harness` will find one; +* the contract, run once per registered harness against the recorded output in + `tests/fixtures/harness//`, so a new harness is held to the same bar as + the first without anyone writing its tests from scratch; +* the seam, which drives the engine with a harness whose events look nothing + like Claude Code's, because a contract only the one real harness has ever + satisfied has not been shown to be a contract. +""" + +from __future__ import annotations + +import json +import os +import shutil +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from skillscope import harness, routing +from skillscope.engine import no_sandbox + +FIXTURES = Path(__file__).parent / "fixtures" / "harness" + + +class TestTheRegistry(unittest.TestCase): + def test_claude_code_is_the_default(self) -> None: + self.assertEqual(harness.DEFAULT, "claude-code") + self.assertEqual(harness.get().name, "claude-code") + self.assertIs(harness.get(), harness.get("claude-code")) + + def test_names_lists_what_is_registered(self) -> None: + self.assertIn("claude-code", harness.names()) + + def test_an_unknown_harness_is_refused_with_the_known_ones(self) -> None: + with self.assertRaises(SystemExit) as raised: + harness.get("no-such-agent") + message = str(raised.exception) + self.assertIn("no-such-agent", message) + self.assertIn("claude-code", message) + + def test_a_name_cannot_be_registered_twice(self) -> None: + with self.assertRaises(ValueError): + harness.register(harness.ClaudeCode()) + + +class TestClaudeCodeIsWhatTheEngineAlwaysRan(unittest.TestCase): + """The adapter replaced inline code, so pin the output it replaced. + + The command line and environment are the whole of what changed hands, and + nothing else in the suite compares them to the earlier values: the + end-to-end test runs a stub that ignores its flags. + """ + + def spec(self, **overrides) -> harness.RunSpec: + values = {"workspace": "/ws", "prompt": "do it", **overrides} + return harness.RunSpec(**values) + + def command(self, **overrides) -> list[str]: + with mock.patch("shutil.which", lambda _: "/usr/bin/claude"): + return harness.get("claude-code").command(self.spec(**overrides)) + + def test_the_command_is_the_one_the_engine_built_inline(self) -> None: + self.assertEqual( + self.command(model="opus", effort="high"), + [ + "/usr/bin/claude", "-p", + "--output-format", "stream-json", "--verbose", + "--dangerously-skip-permissions", + "--add-dir", "/ws", + "--model", "opus", + "--effort", "high", + ], + ) + + def test_model_and_effort_are_left_out_when_not_given(self) -> None: + command = self.command() + self.assertNotIn("--model", command) + self.assertNotIn("--effort", command) + + def test_the_callers_flags_come_last_in_order(self) -> None: + command = self.command(extra_flags=("--max-budget-usd", "1")) + self.assertEqual(command[-2:], ["--max-budget-usd", "1"]) + + def test_a_config_dir_is_only_exported_when_one_was_given(self) -> None: + claude = harness.get("claude-code") + self.assertEqual( + claude.env(self.spec(config_dir=Path("/cfg")))["CLAUDE_CONFIG_DIR"], + str(Path("/cfg")), + ) + with mock.patch.dict(os.environ, clear=False): + os.environ.pop("CLAUDE_CONFIG_DIR", None) + self.assertNotIn("CLAUDE_CONFIG_DIR", claude.env(self.spec())) + + def test_the_cli_is_told_not_to_retry_unless_the_caller_says_otherwise(self) -> None: + claude = harness.get("claude-code") + with mock.patch.dict(os.environ, clear=False): + os.environ.pop("CLAUDE_CODE_MAX_RETRIES", None) + self.assertEqual(claude.env(self.spec())["CLAUDE_CODE_MAX_RETRIES"], "0") + os.environ["CLAUDE_CODE_MAX_RETRIES"] = "3" + self.assertEqual(claude.env(self.spec())["CLAUDE_CODE_MAX_RETRIES"], "3") + + def test_a_missing_binary_is_refused_by_name(self) -> None: + with mock.patch("shutil.which", lambda _: None), self.assertRaises(SystemExit) as raised: + harness.get("claude-code").require() + self.assertEqual(str(raised.exception), "error: 'claude' CLI not found on PATH") + + +class TestEveryRegisteredHarnessMeetsTheContract(unittest.TestCase): + """Run for each harness that ships fixtures; a harness without them fails. + + A fixture directory holds `trigger.jsonl` (the harness activated the skill + and finished), `no_trigger.jsonl` (it finished without activating one) and + `expected.json` with what the engine should read out of each. Real + recordings, token-scrubbed, with the CLI version noted in `expected.json`; + a harness whose recordings change shape gets new fixtures, not a looser + test. + """ + + def fixtures(self, name: str) -> tuple[dict, dict[str, list[str]]]: + folder = FIXTURES / name + self.assertTrue( + folder.is_dir(), + f"harness {name!r} is registered but has no fixtures in {folder}", + ) + expected = json.loads((folder / "expected.json").read_text(encoding="utf-8")) + runs = { + kind: (folder / f"{kind}.jsonl").read_text(encoding="utf-8").splitlines() + for kind in ("trigger", "no_trigger") + } + return expected, runs + + def canonical(self, subject: harness.Harness, lines: list[str]) -> list[dict]: + events: list[dict] = [] + for line in lines: + if line.strip(): + events.extend(subject.normalize(json.loads(line))) + return events + + def test_what_the_engine_reads_out_of_a_recorded_run(self) -> None: + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + expected, runs = self.fixtures(name) + for kind, lines in runs.items(): + events = self.canonical(subject, lines) + want = expected[kind] + + activation = next( + ( + hit + for event in events + if (hit := routing.detect_activation( + event, expected["skills"], + allow_body_path=not subject.has_skill_tool, + )) + ), + None, + ) + self.assertEqual(activation, want["activation"], kind) + + completed, final = no_sandbox.run_completed(events) + self.assertEqual(completed, want["completed"], kind) + self.assertEqual(final, want["final"], kind) + + def test_the_room_is_read_off_the_init_event_when_the_harness_has_one(self) -> None: + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + expected, runs = self.fixtures(name) + events = self.canonical(subject, runs["trigger"]) + seen = [ + visible + for event in events + if (visible := routing.init_skills(event, expected["skills"])) is not None + ] + # A harness with no init event reports nothing, which the engine + # words as "the check did not run". It must never report a room + # it did not read. + for visible in seen: + self.assertEqual(visible, expected["skills"]) + + def test_the_command_carries_the_model_the_effort_and_no_secret(self) -> None: + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + spec = harness.RunSpec( + workspace=str(Path(tempfile.gettempdir()) / "ws"), + prompt="the prompt", + model="a-model", + effort="high", + ) + with mock.patch.dict(os.environ, {"SOME_API_KEY": "sk-secret-value"}): + with mock.patch("shutil.which", lambda b: f"/bin/{b}"): + command = subject.command(spec) + self.assertIn("a-model", command) + self.assertIn("high", command) + self.assertFalse( + any("sk-secret-value" in part for part in command), + "a credential on the command line shows up in `ps` and in logs", + ) + + def test_the_config_is_isolated_from_the_runners_own(self) -> None: + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + spec = harness.RunSpec( + workspace="ws", prompt="p", config_dir=Path("throwaway-config"), + ) + env = subject.env(spec) + self.assertIn( + str(Path("throwaway-config")), env.values(), + "no environment variable points the harness at the throwaway " + "config dir, so the runner's own skills would join the room", + ) + + def test_a_skill_is_staged_where_the_harness_looks(self) -> None: + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + with tempfile.TemporaryDirectory() as tmp: + skill = Path(tmp) / "alpha" + (skill / "evals").mkdir(parents=True) + (skill / "SKILL.md").write_text("# alpha", encoding="utf-8") + (skill / "evals" / "evals.json").write_text("{}", encoding="utf-8") + workspace = Path(tmp) / "ws" + workspace.mkdir() + + no_sandbox.install_skill(skill, str(workspace), subject.skills_dir) + + staged = workspace / subject.skills_dir / "alpha" + self.assertTrue((staged / "SKILL.md").is_file()) + # The answer key stays out of the room for every harness. + self.assertFalse((staged / "evals").exists()) + + +# What the foreign harness below prints. Deliberately nothing like stream-json: +# no `type` keys, different names for everything. +_FOREIGN_AGENT = r""" +import json, os, sys + +def emit(**event): + print(json.dumps(event), flush=True) + +staged = os.path.isfile(os.path.join(".foreign", "skills", "alpha", "SKILL.md")) +# What the harness was handed: its prompt as an argument, and a stdin that was +# closed without ever being written to. +stdin_seen = sys.stdin.read() +emit(evt="started") +emit(evt="skill", name="alpha") +emit( + evt="command", + command="echo foreign-marker staged=%s stdin=%r arg=%r" % (staged, stdin_seen, sys.argv[1]), +) +emit(evt="finished", text="all done") +""" + + +class ForeignHarness(harness.Harness): + """A harness whose events share nothing with Claude Code's. + + Not registered: it exists to show the engine only needs what `Harness` + promises. It runs a script, takes its prompt as an argument, and keeps its + skills somewhere other than `.claude/skills`. + """ + + name = "foreign" + binary = Path(sys.executable).name + skills_dir = ".foreign/skills" + prompt_on_stdin = False + + def __init__(self, script: Path) -> None: + self.script = script + + def require(self) -> None: + return None + + def command(self, spec: harness.RunSpec) -> list[str]: + return [sys.executable, "-u", str(self.script), spec.prompt] + + def env(self, spec: harness.RunSpec) -> dict[str, str]: + return dict(os.environ) + + def normalize(self, event: dict) -> list[dict]: + kind = event.get("evt") + if kind == "skill": + return [_tool_use("Skill", {"skill": event["name"]})] + if kind == "command": + return [_tool_use("Bash", {"command": event["command"]})] + if kind == "finished": + return [{"type": "result", "subtype": "success", "result": event["text"]}] + return [] + + +def _tool_use(name: str, arguments: dict) -> dict: + return { + "type": "assistant", + "message": {"content": [{"type": "tool_use", "name": name, "input": arguments}]}, + } + + +class TestTheSeamTranslatesAnotherHarnessesEvents(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.tmp = tempfile.mkdtemp(prefix="ss-foreign-") + self.addCleanup(shutil.rmtree, self.tmp, True) + self.script = Path(self.tmp) / "foreign_agent.py" + self.script.write_text(_FOREIGN_AGENT, encoding="utf-8") + self.foreign = ForeignHarness(self.script) + + async def stream(self, stop_when) -> tuple[list[dict], str | None]: + spec = harness.RunSpec(workspace=self.tmp, prompt="go") + events, reason, _, _ = await no_sandbox._stream_until( + self.foreign.command(spec), "", self.tmp, self.foreign.env(spec), + stop_when, self.foreign.normalize, + ) + return events, reason + + async def test_the_engine_reads_canonical_events_only(self) -> None: + events, reason = await self.stream(lambda event: None) + self.assertIsNone(reason) + # `started` carries nothing the engine reads, so it is dropped rather + # than passed through for something downstream to trip on. + calls = [call for event in events for call in routing._iter_tool_uses(event)] + self.assertEqual([name for name, _ in calls], ["Skill", "Bash"]) + self.assertEqual(calls[0][1], '{"skill": "alpha"}') + self.assertIn("foreign-marker", calls[1][1]) + completed, final = no_sandbox.run_completed(events) + self.assertEqual((completed, final), (True, "all done")) + + async def test_the_stop_rule_sees_canonical_events(self) -> None: + seen: list[dict] = [] + + def stop_when(event: dict) -> str | None: + seen.append(event) + return "activated" if routing.detect_activation(event, ["alpha"]) else None + + events, reason = await self.stream(stop_when) + self.assertEqual(reason, "activated") + self.assertEqual(len(events), 1) + self.assertTrue(all("evt" not in event for event in seen)) + + async def test_an_unreadable_line_is_skipped_not_fatal(self) -> None: + self.script.write_text( + "import json\nprint('not json', flush=True)\n" + "print(json.dumps({'evt': 'finished', 'text': 'ok'}), flush=True)\n", + encoding="utf-8", + ) + events, _ = await self.stream(lambda event: None) + self.assertEqual(no_sandbox.run_completed(events), (True, "ok")) + + +class TestTheEngineRunsACaseWithAnotherHarness(unittest.TestCase): + """The whole solver, under inspect, with a harness that is not Claude Code.""" + + def setUp(self) -> None: + try: + import inspect_ai # noqa: F401 + except ImportError: + self.skipTest("needs inspect_ai") + from skillscope.engine import sandbox as engine_sandbox + + env = mock.patch.dict(os.environ, {engine_sandbox.SANDBOX_ENV: "local"}) + env.start() + self.addCleanup(env.stop) + self.tmp = tempfile.mkdtemp(prefix="ss-foreign-e2e-") + self.addCleanup(shutil.rmtree, self.tmp, True) + self.script = Path(self.tmp) / "foreign_agent.py" + self.script.write_text(_FOREIGN_AGENT, encoding="utf-8") + self.skill = Path(self.tmp) / "alpha" + self.skill.mkdir() + (self.skill / "SKILL.md").write_text( + "---\nname: alpha\ndescription: does alpha things\n---\n", encoding="utf-8" + ) + + def run_case(self, *, stop_when_factory=None): + import contextlib + import io + + from inspect_ai import Task + from inspect_ai import eval as inspect_eval + from inspect_ai.dataset import Sample + + task = Task( + dataset=[Sample(id="c1", input="do the alpha thing")], + solver=no_sandbox.harness_solver( + ForeignHarness(self.script), "mockllm/model", None, self.skill, + stop_when_factory=stop_when_factory, + ), + sandbox="local", + ) + with contextlib.redirect_stdout(io.StringIO()): + [log] = inspect_eval( + task, model="mockllm/model", log_dir=str(Path(self.tmp) / "logs"), + display="plain", + ) + [sample] = log.samples + self.assertIsNone(sample.error, sample.error) + return sample + + def calls(self, sample) -> list[tuple[str, dict]]: + return [ + (call.function, call.arguments) + for message in sample.messages + for call in getattr(message, "tool_calls", None) or [] + ] + + def test_the_foreign_events_become_tool_calls_the_scorers_read(self) -> None: + sample = self.run_case() + functions = [function for function, _ in self.calls(sample)] + self.assertEqual(functions, ["Skill", "Bash"]) + self.assertIn("all done", sample.output.completion) + + def test_the_skill_is_staged_where_that_harness_looks(self) -> None: + # The agent script checks for the file itself, in the workspace it was + # started in, and reports what it found. + sample = self.run_case() + [(_, command)] = [call for call in self.calls(sample) if call[0] == "Bash"] + self.assertIn("staged=True", command["command"]) + + def test_a_prompt_taken_as_an_argument_is_not_also_written_to_stdin(self) -> None: + sample = self.run_case() + [(_, command)] = [call for call in self.calls(sample) if call[0] == "Bash"] + self.assertIn("arg='do the alpha thing'", command["command"]) + # Closed and empty, not left open: a CLI that waits on stdin would hang. + self.assertIn("stdin=''", command["command"]) + + def test_the_stop_rule_ends_the_run_on_a_foreign_activation(self) -> None: + def factory(): + return lambda event: ( + "activated" if routing.detect_activation(event, ["alpha"]) else None + ) + + sample = self.run_case(stop_when_factory=factory) + self.assertEqual([function for function, _ in self.calls(sample)], ["Skill"]) + + +if __name__ == "__main__": + unittest.main() From fc1e678df99b84621fe15152fa51a96aeee7a52b Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Thu, 29 Oct 2026 12:50:09 +0200 Subject: [PATCH 2/8] Let a harness keep state while it translates one run's events `Harness.normalize` translates each event on its own. That is not enough for an agent whose closing event does not repeat the answer: GitHub Copilot CLI's final `result` record carries only an exit code and usage, and the answer is the last assistant message, printed several events earlier. A translator for such a harness has to remember it. Add `Harness.normalizer()`, which returns a translator for one run and may keep state. The default returns the stateless `normalize`, so Claude Code is unchanged. The engine asks for a fresh one per case, so nothing remembered in one case reaches the next, and the state lives in the returned callable rather than on the harness instance, which every concurrent case shares. The test harness now behaves like that: its closing event carries no answer. New tests check that each translator has its own memory, that two runs translated at once do not share it (for every registered harness, from the recorded fixtures), and that a case's answer does not leak into the next one when the solver runs two cases in a row under inspect. Breaking the code two ways -- sharing one translator across cases, and bypassing it on the non-streaming path -- makes those tests fail. Signed-off-by: Eugene Volen --- docs/harnesses.md | 12 ++-- skillscope/engine/no_sandbox.py | 7 ++- skillscope/harness/base.py | 18 ++++++ tests/test_harness.py | 98 ++++++++++++++++++++++++++++----- 4 files changed, 116 insertions(+), 19 deletions(-) diff --git a/docs/harnesses.md b/docs/harnesses.md index 59dd215..7ff0e47 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -31,7 +31,8 @@ A harness is a subclass of `skillscope.harness.Harness`, registered in | `require()` | Refuse to start when the harness is not installed. The default checks `binary` is on `PATH`. | | `command(spec)` | The argv that runs one case, headless, streaming JSON events. | | `env(spec)` | The environment of that process, including whatever points it at `spec.config_dir` so the runner's own settings and skills stay out of the session. | -| `normalize(event)` | Turns one event the harness printed into canonical events (below). | +| `normalize(event)` | Turns one event the harness printed into canonical events (below). Stateless. | +| `normalizer()` | A translator for one run's events, which may keep state. The engine asks for a fresh one per case. Override it when an event cannot be translated on its own, for instance when the closing event does not repeat the answer and it has to be remembered from an earlier one. The default returns `normalize`. | `spec` is a `RunSpec`: the workspace, the prompt, the model and effort, an optional throwaway config directory, and any spend or session flags the caller @@ -51,7 +52,9 @@ Claude Code's stream-json, reduced to what is read: A harness whose stream already has this shape keeps the default `normalize`, which passes events through. Any other maps its events onto these and returns an -empty list for events that carry nothing the engines read. Every stop rule, +empty list for events that carry nothing the engines read. What a harness remembers about a stream belongs in the +translator `normalizer()` returns, never on the harness itself: one harness +instance serves every case in a run, and cases run concurrently. Every stop rule, budget, score and judge sees canonical events only, never the raw ones, so each is written once for every harness. @@ -68,8 +71,9 @@ is written once for every harness. 3. Run `python -m unittest tests.test_harness`. The contract test runs once per registered harness and fails if there are no fixtures. It checks activation, completion, the closing answer, that the command carries the model and effort - and no credential, that the config directory is isolated, and that a skill - is staged where the harness looks and without its `evals/` directory. + and no credential, that the config directory is isolated, that two runs translated at the same time + do not share memory, and that a skill is staged where the harness looks and + without its `evals/` directory. 4. Say in the pull request how the harness differs from the others in ways a reader of a report should know: how it activates skills, what it cannot report, and what it needs to authenticate. diff --git a/skillscope/engine/no_sandbox.py b/skillscope/engine/no_sandbox.py index e41b8d6..f4091ad 100644 --- a/skillscope/engine/no_sandbox.py +++ b/skillscope/engine/no_sandbox.py @@ -485,6 +485,9 @@ async def solve(state, generate): # Closed either way: a CLI that is not given its prompt on stdin # can still sit waiting on it. stdin_text = prompt if harness.prompt_on_stdin else "" + # One per case, not one per run: it may remember what it has read, + # and the next case must not inherit that. + normalize = harness.normalizer() events: list[dict] = [] if stop_when_factory is None: @@ -497,14 +500,14 @@ async def solve(state, generate): if not line: continue try: - events.extend(harness.normalize(json.loads(line))) + events.extend(normalize(json.loads(line))) except json.JSONDecodeError: continue else: async with _subprocess_concurrency(): events, stop_reason, returncode, stderr = await _stream_until( cmd, stdin_text, workspace, env, stop_when_factory(), - harness.normalize, + normalize, ) # Recorded whatever it was, including `None` for a stream that # ended on its own. A stop this driver performed leaves no diff --git a/skillscope/harness/base.py b/skillscope/harness/base.py index 32c25aa..d3034a7 100644 --- a/skillscope/harness/base.py +++ b/skillscope/harness/base.py @@ -25,6 +25,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from pathlib import Path +from typing import Callable @dataclass(frozen=True) @@ -88,5 +89,22 @@ def normalize(self, event: dict) -> list[dict]: final `result` event. A harness whose stream already has that shape keeps this default; any other maps its events onto it, and returns an empty list for the ones that carry nothing the engines read. + + Stateless, which is enough when each event can be translated on its + own. A harness that cannot -- one whose closing event does not repeat + the answer, so it has to be remembered from an earlier one -- overrides + `normalizer` instead. """ return [event] + + def normalizer(self) -> Callable[[dict], list[dict]]: + """A translator for the events of one run, free to keep state. + + The engine asks for a fresh one per case, so nothing a harness + remembers can leak from one case into the next. A harness instance is + shared by every case in a run, and cases run concurrently; the state + therefore lives in the returned callable, never on `self`. + + The default is the stateless `normalize`. + """ + return self.normalize diff --git a/tests/test_harness.py b/tests/test_harness.py index 56188e8..0913e9b 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -141,9 +141,10 @@ def fixtures(self, name: str) -> tuple[dict, dict[str, list[str]]]: def canonical(self, subject: harness.Harness, lines: list[str]) -> list[dict]: events: list[dict] = [] + normalize = subject.normalizer() for line in lines: if line.strip(): - events.extend(subject.normalize(json.loads(line))) + events.extend(normalize(json.loads(line))) return events def test_what_the_engine_reads_out_of_a_recorded_run(self) -> None: @@ -189,6 +190,27 @@ def test_the_room_is_read_off_the_init_event_when_the_harness_has_one(self) -> N for visible in seen: self.assertEqual(visible, expected["skills"]) + def test_two_runs_translated_at_once_do_not_share_memory(self) -> None: + # The engine runs cases concurrently against one harness instance, so + # whatever a harness remembers about a stream must live in the + # translator it hands out per run, not on the harness. + import itertools + + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + _, runs = self.fixtures(name) + alone = {kind: self.canonical(subject, lines) for kind, lines in runs.items()} + + first, second = subject.normalizer(), subject.normalizer() + mixed: dict[str, list[dict]] = {"trigger": [], "no_trigger": []} + for a, b in itertools.zip_longest(runs["trigger"], runs["no_trigger"]): + if a and a.strip(): + mixed["trigger"].extend(first(json.loads(a))) + if b and b.strip(): + mixed["no_trigger"].extend(second(json.loads(b))) + self.assertEqual(mixed, alone) + def test_the_command_carries_the_model_the_effort_and_no_secret(self) -> None: for name in harness.names(): with self.subTest(harness=name): @@ -256,12 +278,14 @@ def emit(**event): # closed without ever being written to. stdin_seen = sys.stdin.read() emit(evt="started") +if "silent" not in sys.argv[1]: + emit(evt="said", text="all done") emit(evt="skill", name="alpha") emit( evt="command", command="echo foreign-marker staged=%s stdin=%r arg=%r" % (staged, stdin_seen, sys.argv[1]), ) -emit(evt="finished", text="all done") +emit(evt="finished") """ @@ -290,15 +314,25 @@ def command(self, spec: harness.RunSpec) -> list[str]: def env(self, spec: harness.RunSpec) -> dict[str, str]: return dict(os.environ) - def normalize(self, event: dict) -> list[dict]: - kind = event.get("evt") - if kind == "skill": - return [_tool_use("Skill", {"skill": event["name"]})] - if kind == "command": - return [_tool_use("Bash", {"command": event["command"]})] - if kind == "finished": - return [{"type": "result", "subtype": "success", "result": event["text"]}] - return [] + def normalizer(self): + # Its closing event does not repeat the answer, so it is remembered from + # the one that said it -- which is why the translator is per run. + said = {"text": ""} + + def normalize(event: dict) -> list[dict]: + kind = event.get("evt") + if kind == "said": + said["text"] = event["text"] + return [] + if kind == "skill": + return [_tool_use("Skill", {"skill": event["name"]})] + if kind == "command": + return [_tool_use("Bash", {"command": event["command"]})] + if kind == "finished": + return [{"type": "result", "subtype": "success", "result": said["text"]}] + return [] + + return normalize def _tool_use(name: str, arguments: dict) -> dict: @@ -320,7 +354,7 @@ async def stream(self, stop_when) -> tuple[list[dict], str | None]: spec = harness.RunSpec(workspace=self.tmp, prompt="go") events, reason, _, _ = await no_sandbox._stream_until( self.foreign.command(spec), "", self.tmp, self.foreign.env(spec), - stop_when, self.foreign.normalize, + stop_when, self.foreign.normalizer(), ) return events, reason @@ -336,6 +370,14 @@ async def test_the_engine_reads_canonical_events_only(self) -> None: completed, final = no_sandbox.run_completed(events) self.assertEqual((completed, final), (True, "all done")) + def test_each_translator_has_its_own_memory(self) -> None: + first, second = self.foreign.normalizer(), self.foreign.normalizer() + first({"evt": "said", "text": "first run"}) + [unspoken] = second({"evt": "finished"}) + self.assertEqual(unspoken["result"], "") + [spoken] = first({"evt": "finished"}) + self.assertEqual(spoken["result"], "first run") + async def test_the_stop_rule_sees_canonical_events(self) -> None: seen: list[dict] = [] @@ -351,7 +393,8 @@ def stop_when(event: dict) -> str | None: async def test_an_unreadable_line_is_skipped_not_fatal(self) -> None: self.script.write_text( "import json\nprint('not json', flush=True)\n" - "print(json.dumps({'evt': 'finished', 'text': 'ok'}), flush=True)\n", + "print(json.dumps({'evt': 'said', 'text': 'ok'}), flush=True)\n" + "print(json.dumps({'evt': 'finished'}), flush=True)\n", encoding="utf-8", ) events, _ = await self.stream(lambda event: None) @@ -419,6 +462,35 @@ def test_the_foreign_events_become_tool_calls_the_scorers_read(self) -> None: self.assertEqual(functions, ["Skill", "Bash"]) self.assertIn("all done", sample.output.completion) + def test_one_cases_answer_does_not_leak_into_the_next(self) -> None: + import contextlib + import io + + from inspect_ai import Task + from inspect_ai import eval as inspect_eval + from inspect_ai.dataset import Sample + + task = Task( + dataset=[ + Sample(id="talks", input="do the alpha thing"), + Sample(id="silent", input="be silent about the alpha thing"), + ], + solver=no_sandbox.harness_solver( + ForeignHarness(self.script), "mockllm/model", None, self.skill, + ), + sandbox="local", + ) + with contextlib.redirect_stdout(io.StringIO()): + [log] = inspect_eval( + task, model="mockllm/model", log_dir=str(Path(self.tmp) / "logs"), + display="plain", + # One at a time, so the second case really does follow the first. + max_samples=1, + ) + answers = {sample.id: sample.output.completion for sample in log.samples} + self.assertEqual(answers["talks"], "all done") + self.assertEqual(answers["silent"], "(no final message)") + def test_the_skill_is_staged_where_that_harness_looks(self) -> None: # The agent script checks for the file itself, in the workspace it was # started in, and reports what it found. From 96d63b9fd730f921ec203278f95e1f95d30a2433 Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Fri, 30 Oct 2026 12:43:21 +0200 Subject: [PATCH 3/8] Keep the harness neutral about where it runs A harness should define an agent once, for the host and for a container alike, so that a sandbox leg can start the same command instead of delegating to a second definition of the same agent. Four things in the contract assumed the host. `command()` named an absolute path found on the runner. It now emits the executable's name, and `host_command()` resolves it on this machine, so the same command can be started anywhere. `env()` copied the whole runner environment, so a container would have inherited PATH, HOME and every key on the machine. It now returns only what the harness sets, and `host_env()` puts that on top of `os.environ`, which leaves host behaviour as it was: the runner's own CLAUDE_CODE_MAX_RETRIES still overrides the default, and the isolation setting still wins over the runner's own CLAUDE_CONFIG_DIR. `RunSpec.config_dir` was a host Path and is now a string naming a path wherever the agent runs, as `workspace` already was. `require()` is documented as the host check. docs/harnesses.md now says that a harness defines the agent once for both legs, and lists the sandbox leg under what is not yet behind the contract. New tests run for every registered harness: the command names its executable and does not locate it, the environment carries only what the harness sets, and on the host that goes on top of the runner's. The Claude Code golden checks compare the host command and environment with what the engine built inline. The engine hands the harness strings, not host Paths. Each of the five changes was broken in turn and caught. Signed-off-by: Eugene Volen --- docs/harnesses.md | 25 +++++-- skillscope/agent.py | 16 ++-- skillscope/engine/no_sandbox.py | 8 +- skillscope/harness/base.py | 44 +++++++++-- skillscope/harness/claude_code.py | 8 +- tests/test_harness.py | 118 +++++++++++++++++++++++++----- 6 files changed, 179 insertions(+), 40 deletions(-) diff --git a/docs/harnesses.md b/docs/harnesses.md index 7ff0e47..67818cb 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -13,6 +13,14 @@ a container); the harness says *what*. The engines drive a harness as a subprocess, so the contract between them is small, and this page is that contract. +**A harness defines the agent once, for both answers to *where*.** On the host the +engine starts `command(spec)` as a subprocess: the executable resolved on the host, +and the runner's environment underneath what `env(spec)` sets. A sandbox leg starts +the same command inside the container, rather than delegating to a separate agent +definition such as an `inspect_swe` one. Otherwise every agent would exist twice in +this repo, with its own command line, environment, skill staging and event handling, +free to drift apart. + Today one harness is registered, `claude-code`. The contract exists so that a second one is an adapter and a set of tests rather than a change to the engines. @@ -24,19 +32,21 @@ A harness is a subclass of `skillscope.harness.Harness`, registered in | Member | What it is | | --- | --- | | `name` | The name reports and `--harness` use. | -| `binary` | The executable looked up on `PATH`. | +| `binary` | The executable's name. Whoever starts the process resolves it: on the host with `shutil.which`, in a container by the guest's `PATH`. | | `skills_dir` | Where, relative to the workspace, the harness discovers skills (`.claude/skills` for Claude Code). The engine stages each skill there. | | `has_skill_tool` | Whether activating a skill is a tool call in the stream. A harness that activates a skill by reading its `SKILL.md` sets this `False`, and routing reads activation from the file access instead. | | `prompt_on_stdin` | Whether the prompt is written to stdin. A harness that takes it as an argument builds it into `command` and sets this `False`; stdin is closed either way, because some CLIs wait on it. | -| `require()` | Refuse to start when the harness is not installed. The default checks `binary` is on `PATH`. | -| `command(spec)` | The argv that runs one case, headless, streaming JSON events. | -| `env(spec)` | The environment of that process, including whatever points it at `spec.config_dir` so the runner's own settings and skills stay out of the session. | +| `require()` | Refuse to start **on the host** when the harness is not installed there. The default checks `binary` is on the runner's `PATH`. A leg that starts the agent in a container checks the image instead. | +| `command(spec)` | The argv that runs one case, headless, streaming JSON events. The executable is named (`binary`), never an absolute path on this machine, so the same command can be started on the host or in a container. | +| `env(spec)` | What the harness sets in the process's environment, and nothing else: its own settings, and whatever points it at `spec.config_dir` so the runner's own settings and skills stay out of the session. Not the runner's environment (`PATH`, `HOME`, every key on the machine), which a container must not inherit. Whoever starts the process puts this on top of the environment it has. | +| `host_command(spec)`, `host_env(spec)` | What a run started on the host executes: `command(spec)` with its executable found by `shutil.which` (on Windows an npm-installed CLI is a `.cmd` shim that only that finds), and `os.environ` with `env(spec)` on top. A sandbox leg uses `command` and `env` directly. | | `normalize(event)` | Turns one event the harness printed into canonical events (below). Stateless. | | `normalizer()` | A translator for one run's events, which may keep state. The engine asks for a fresh one per case. Override it when an event cannot be translated on its own, for instance when the closing event does not repeat the answer and it has to be remembered from an earlier one. The default returns `normalize`. | `spec` is a `RunSpec`: the workspace, the prompt, the model and effort, an optional throwaway config directory, and any spend or session flags the caller -built. +built. The workspace and the config directory are strings naming a path wherever +the agent runs: a host path on the host, a path inside the container in a sandbox. ## The canonical event form @@ -93,5 +103,10 @@ still assume Claude Code, and a second harness will need to move them behind it: * the spend-limit flags, which are probed from `claude --help`; * the CI model pin, which forces `opus`; * the judge, which uses an Anthropic model through `inspect_ai`; +* **the sandbox leg**: `--engine claude-code` still runs Claude Code through + `inspect_swe`, with its own command line, environment, skill staging and event + handling. It should start the harness's `command(spec)` in the container, so that + each agent is defined once. Until it does, a harness other than Claude Code has no + sandboxed leg; * `.claude/skills` as the path routing treats as a skills inspection, rather than a harness's own `skills_dir`. diff --git a/skillscope/agent.py b/skillscope/agent.py index c37fc3c..8a2c551 100644 --- a/skillscope/agent.py +++ b/skillscope/agent.py @@ -77,17 +77,23 @@ def enforce_model_policy(model: str | None) -> str | None: return AUTOMATED_MODEL -def claude_env() -> dict[str, str]: - """Environment for `claude` subprocesses. +def claude_settings() -> dict[str, str]: + """What `claude` is told, beyond whatever environment it is started in. Disable the CLI's internal retry loop by default so a network/auth problem (e.g. not connected to the network that can reach the API) fails fast instead of being retried into a long, confusing hang. The caller can still override by exporting ``CLAUDE_CODE_MAX_RETRIES``. + + Only the settings, not the runner's environment: where the CLI runs decides + what else it is given. """ - env = dict(os.environ) - env.setdefault("CLAUDE_CODE_MAX_RETRIES", "0") - return env + return {"CLAUDE_CODE_MAX_RETRIES": os.environ.get("CLAUDE_CODE_MAX_RETRIES", "0")} + + +def claude_env() -> dict[str, str]: + """Environment for `claude` subprocesses started on this machine.""" + return {**os.environ, **claude_settings()} def check_api_reachable(model: str | None = DEFAULT_MODEL, timeout: float = 60) -> tuple[bool, str]: diff --git a/skillscope/engine/no_sandbox.py b/skillscope/engine/no_sandbox.py index f4091ad..e614ede 100644 --- a/skillscope/engine/no_sandbox.py +++ b/skillscope/engine/no_sandbox.py @@ -477,11 +477,13 @@ async def solve(state, generate): prompt=prompt, model=model, effort=effort, - config_dir=config_dir, + config_dir=str(config_dir) if config_dir is not None else None, extra_flags=tuple(extra_flags or ()), ) - cmd = harness.command(spec) - env = harness.env(spec) + # Started on the host, so the executable is looked up here and the + # runner's environment goes under what the harness sets. + cmd = harness.host_command(spec) + env = harness.host_env(spec) # Closed either way: a CLI that is not given its prompt on stdin # can still sit waiting on it. stdin_text = prompt if harness.prompt_on_stdin else "" diff --git a/skillscope/harness/base.py b/skillscope/harness/base.py index d3034a7..b01d6da 100644 --- a/skillscope/harness/base.py +++ b/skillscope/harness/base.py @@ -21,10 +21,10 @@ from __future__ import annotations +import os import shutil from abc import ABC, abstractmethod from dataclasses import dataclass -from pathlib import Path from typing import Callable @@ -43,7 +43,9 @@ class RunSpec: effort: str | None = None # A throwaway config directory, so the runner's own settings and user-level # skills stay out of the session. `None` when the caller does not isolate. - config_dir: Path | None = None + # A path wherever the agent runs, like `workspace`: on the host a host path, + # in a container a path inside it. + config_dir: str | None = None # The harness's own spend or session controls, already built by the caller. extra_flags: tuple[str, ...] = () @@ -67,17 +69,49 @@ class Harness(ABC): prompt_on_stdin: bool = True def require(self) -> None: - """Refuse to start when the harness is not installed.""" + """Refuse to start on the host when the harness is not installed there. + + This asks the runner's `PATH`, which is the right question for a run + started on the host. A leg that starts the agent in a container asks + about the image instead. + """ if not shutil.which(self.binary): raise SystemExit(f"error: '{self.binary}' CLI not found on PATH") @abstractmethod def command(self, spec: RunSpec) -> list[str]: - """The argv that runs one case, headless, streaming JSON events.""" + """The argv that runs one case, headless, streaming JSON events. + + The executable is named, not located: `self.binary`, never an absolute + path on this machine. Whoever starts the process resolves it, so the same + command can be started on the host or inside a container. + """ @abstractmethod def env(self, spec: RunSpec) -> dict[str, str]: - """The environment that process gets, config isolation included.""" + """What the harness sets in the process's environment, and nothing else. + + Its own settings, and whatever points it at `spec.config_dir` so the + runner's own settings and skills stay out of the session. Not the runner's + environment: `PATH`, `HOME` and every key on the machine are not the + harness's to carry, and a container must not inherit them. Whoever starts + the process puts this on top of the environment it has. + """ + + def host_command(self, spec: RunSpec) -> list[str]: + """`command(spec)` with its executable resolved on this machine. + + What a run started on the host executes. The lookup is `shutil.which` + rather than leaving it to the operating system, because on Windows a CLI + installed with npm is a `.cmd` shim that only the former finds. + """ + argv = self.command(spec) + return [shutil.which(argv[0]) or argv[0], *argv[1:]] + + def host_env(self, spec: RunSpec) -> dict[str, str]: + """The environment a run started on the host gets: this machine's, with + what the harness sets on top.""" + return {**os.environ, **self.env(spec)} def normalize(self, event: dict) -> list[dict]: """Translate one event the harness printed into canonical events. diff --git a/skillscope/harness/claude_code.py b/skillscope/harness/claude_code.py index edc659d..9cd89c0 100644 --- a/skillscope/harness/claude_code.py +++ b/skillscope/harness/claude_code.py @@ -12,8 +12,6 @@ from __future__ import annotations -import shutil - from .. import agent from .base import Harness, RunSpec @@ -25,7 +23,7 @@ class ClaudeCode(Harness): def command(self, spec: RunSpec) -> list[str]: cmd = [ - shutil.which(self.binary), "-p", + self.binary, "-p", "--output-format", "stream-json", "--verbose", "--dangerously-skip-permissions", "--add-dir", spec.workspace, @@ -38,7 +36,7 @@ def command(self, spec: RunSpec) -> list[str]: return cmd def env(self, spec: RunSpec) -> dict[str, str]: - env = agent.claude_env() + env = agent.claude_settings() if spec.config_dir is not None: - env["CLAUDE_CONFIG_DIR"] = str(spec.config_dir) + env["CLAUDE_CONFIG_DIR"] = spec.config_dir return env diff --git a/tests/test_harness.py b/tests/test_harness.py index 0913e9b..40c8043 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -66,14 +66,15 @@ def spec(self, **overrides) -> harness.RunSpec: return harness.RunSpec(**values) def command(self, **overrides) -> list[str]: - with mock.patch("shutil.which", lambda _: "/usr/bin/claude"): - return harness.get("claude-code").command(self.spec(**overrides)) + return harness.get("claude-code").command(self.spec(**overrides)) def test_the_command_is_the_one_the_engine_built_inline(self) -> None: + # Naming the executable and not locating it is the one difference from + # what the engine built inline: `host_command` puts the located path back. self.assertEqual( self.command(model="opus", effort="high"), [ - "/usr/bin/claude", "-p", + "claude", "-p", "--output-format", "stream-json", "--verbose", "--dangerously-skip-permissions", "--add-dir", "/ws", @@ -91,15 +92,49 @@ def test_the_callers_flags_come_last_in_order(self) -> None: command = self.command(extra_flags=("--max-budget-usd", "1")) self.assertEqual(command[-2:], ["--max-budget-usd", "1"]) + def test_run_on_the_host_it_is_the_executable_found_here(self) -> None: + claude = harness.get("claude-code") + with mock.patch("shutil.which", lambda _: "/usr/bin/claude"): + started = claude.host_command(self.spec(model="opus", effort="high")) + self.assertEqual(started[0], "/usr/bin/claude") + self.assertEqual(started[1:], self.command(model="opus", effort="high")[1:]) + + def test_an_executable_that_is_not_found_is_left_for_the_start_to_refuse(self) -> None: + with mock.patch("shutil.which", lambda _: None): + started = harness.get("claude-code").host_command(self.spec()) + self.assertEqual(started[0], "claude") + def test_a_config_dir_is_only_exported_when_one_was_given(self) -> None: claude = harness.get("claude-code") self.assertEqual( - claude.env(self.spec(config_dir=Path("/cfg")))["CLAUDE_CONFIG_DIR"], - str(Path("/cfg")), + claude.env(self.spec(config_dir="/cfg"))["CLAUDE_CONFIG_DIR"], "/cfg" ) - with mock.patch.dict(os.environ, clear=False): - os.environ.pop("CLAUDE_CONFIG_DIR", None) - self.assertNotIn("CLAUDE_CONFIG_DIR", claude.env(self.spec())) + self.assertNotIn("CLAUDE_CONFIG_DIR", claude.env(self.spec())) + + def test_the_environment_is_only_what_the_harness_sets(self) -> None: + # Not the runner's: a container must not inherit its PATH, its HOME or + # every key on the machine. + claude = harness.get("claude-code") + ambient = {"PATH": "/runner/bin", "HOME": "/home/runner", "ANTHROPIC_API_KEY": "sk-x"} + with mock.patch.dict(os.environ, ambient): + env = claude.env(self.spec(config_dir="/cfg")) + self.assertEqual(set(env), {"CLAUDE_CODE_MAX_RETRIES", "CLAUDE_CONFIG_DIR"}) + + def test_on_the_host_the_environment_is_what_the_engine_always_built(self) -> None: + # The old inline code: the runner's environment, a retry default that the + # caller can override, and the isolation setting on top of everything. + claude = harness.get("claude-code") + for retries in (None, "3"): + with self.subTest(retries=retries), mock.patch.dict( + os.environ, {"CLAUDE_CONFIG_DIR": "/runners/own"}, clear=False + ): + os.environ.pop("CLAUDE_CODE_MAX_RETRIES", None) + if retries: + os.environ["CLAUDE_CODE_MAX_RETRIES"] = retries + expected = dict(os.environ) + expected.setdefault("CLAUDE_CODE_MAX_RETRIES", "0") + expected["CLAUDE_CONFIG_DIR"] = "/cfg" + self.assertEqual(claude.host_env(self.spec(config_dir="/cfg")), expected) def test_the_cli_is_told_not_to_retry_unless_the_caller_says_otherwise(self) -> None: claude = harness.get("claude-code") @@ -222,8 +257,7 @@ def test_the_command_carries_the_model_the_effort_and_no_secret(self) -> None: effort="high", ) with mock.patch.dict(os.environ, {"SOME_API_KEY": "sk-secret-value"}): - with mock.patch("shutil.which", lambda b: f"/bin/{b}"): - command = subject.command(spec) + command = subject.command(spec) self.assertIn("a-model", command) self.assertIn("high", command) self.assertFalse( @@ -236,15 +270,53 @@ def test_the_config_is_isolated_from_the_runners_own(self) -> None: with self.subTest(harness=name): subject = harness.get(name) spec = harness.RunSpec( - workspace="ws", prompt="p", config_dir=Path("throwaway-config"), + workspace="ws", prompt="p", config_dir="throwaway-config", ) env = subject.env(spec) self.assertIn( - str(Path("throwaway-config")), env.values(), + "throwaway-config", env.values(), "no environment variable points the harness at the throwaway " "config dir, so the runner's own skills would join the room", ) + def test_the_command_names_its_executable_and_does_not_locate_it(self) -> None: + # So the same command can be started on the host or in a container. An + # absolute path on the runner is wrong anywhere else. + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + command = subject.command(harness.RunSpec(workspace="ws", prompt="p")) + self.assertEqual(command[0], subject.binary) + + def test_the_environment_carries_only_what_the_harness_sets(self) -> None: + # Not the runner's PATH, its HOME or any key on it: a container would + # inherit all of it. + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + ambient = { + "PATH": "/runner/bin", "HOME": "/home/runner", + "SOME_API_KEY": "sk-secret-value", "SYSTEMROOT": "C:/Windows", + } + with mock.patch.dict(os.environ, ambient): + env = subject.env(harness.RunSpec( + workspace="ws", prompt="p", config_dir="throwaway-config", + )) + for variable in ambient: + self.assertNotIn(variable, env) + self.assertTrue(all(isinstance(v, str) for v in env.values())) + + def test_on_the_host_what_the_harness_sets_goes_on_top_of_the_runners(self) -> None: + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + spec = harness.RunSpec(workspace="ws", prompt="p", config_dir="cfg") + with mock.patch.dict(os.environ, {"AMBIENT_FOR_THE_TEST": "kept"}): + host = subject.host_env(spec) + self.assertEqual(host["AMBIENT_FOR_THE_TEST"], "kept") + for key, value in subject.env(spec).items(): + self.assertEqual(host[key], value) + def test_a_skill_is_staged_where_the_harness_looks(self) -> None: for name in harness.names(): with self.subTest(harness=name): @@ -304,15 +376,17 @@ class ForeignHarness(harness.Harness): def __init__(self, script: Path) -> None: self.script = script + self.specs: list[harness.RunSpec] = [] def require(self) -> None: return None def command(self, spec: harness.RunSpec) -> list[str]: + self.specs.append(spec) return [sys.executable, "-u", str(self.script), spec.prompt] def env(self, spec: harness.RunSpec) -> dict[str, str]: - return dict(os.environ) + return {} def normalizer(self): # Its closing event does not repeat the answer, so it is remembered from @@ -353,7 +427,7 @@ def setUp(self) -> None: async def stream(self, stop_when) -> tuple[list[dict], str | None]: spec = harness.RunSpec(workspace=self.tmp, prompt="go") events, reason, _, _ = await no_sandbox._stream_until( - self.foreign.command(spec), "", self.tmp, self.foreign.env(spec), + self.foreign.host_command(spec), "", self.tmp, self.foreign.host_env(spec), stop_when, self.foreign.normalizer(), ) return events, reason @@ -418,13 +492,14 @@ def setUp(self) -> None: self.addCleanup(shutil.rmtree, self.tmp, True) self.script = Path(self.tmp) / "foreign_agent.py" self.script.write_text(_FOREIGN_AGENT, encoding="utf-8") + self.foreign = ForeignHarness(self.script) self.skill = Path(self.tmp) / "alpha" self.skill.mkdir() (self.skill / "SKILL.md").write_text( "---\nname: alpha\ndescription: does alpha things\n---\n", encoding="utf-8" ) - def run_case(self, *, stop_when_factory=None): + def run_case(self, *, stop_when_factory=None, config_dir=None): import contextlib import io @@ -435,8 +510,8 @@ def run_case(self, *, stop_when_factory=None): task = Task( dataset=[Sample(id="c1", input="do the alpha thing")], solver=no_sandbox.harness_solver( - ForeignHarness(self.script), "mockllm/model", None, self.skill, - stop_when_factory=stop_when_factory, + self.foreign, "mockllm/model", None, self.skill, + config_dir=config_dir, stop_when_factory=stop_when_factory, ), sandbox="local", ) @@ -491,6 +566,15 @@ def test_one_cases_answer_does_not_leak_into_the_next(self) -> None: self.assertEqual(answers["talks"], "all done") self.assertEqual(answers["silent"], "(no final message)") + def test_the_harness_is_told_where_the_agent_runs_as_strings(self) -> None: + # A path wherever the agent runs, so a container leg can name one inside + # it: the engine hands over strings, not host Path objects. + self.run_case(config_dir=Path(self.tmp) / "cfg") + [spec] = self.foreign.specs + self.assertIsInstance(spec.config_dir, str) + self.assertEqual(spec.config_dir, str(Path(self.tmp) / "cfg")) + self.assertIsInstance(spec.workspace, str) + def test_the_skill_is_staged_where_that_harness_looks(self) -> None: # The agent script checks for the file itself, in the workspace it was # started in, and reports what it found. From 41aaed4e8c603a61e5cf85b71ef11dcc4f08e74e Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Thu, 29 Oct 2026 12:56:28 +0200 Subject: [PATCH 4/8] Add GitHub Copilot CLI as a harness Copilot CLI is the second agent harness. It authenticates from a token in the environment, so a graded run needs no model key, and it discovers skills in `.claude/skills`, where they are already staged. Its events share nothing with Claude Code's, so `Copilot` translates them: `tool.execution_start` and `tool.execution_complete` become a tool call and its result (activating a skill is a call to the tool named `skill`, the call the engines already read; a call the permission rules deny is still recorded), and the closing `result` event, which carries no answer, gets the last assistant message remembered from earlier in the run. Token counts, which the CLI reports once and per model, reach the usage recorder as one usage-only assistant event. Streaming deltas and bookkeeping are dropped. The command runs headless with the prompt on stdin, every permission granted (a non-interactive run cannot answer a prompt) and the built-in MCP servers off; the environment points `COPILOT_HOME` at the throwaway config directory and turns self-update off so the version being graded cannot change mid-run. Fixtures are two real recordings from CLI 1.0.105, one where the skill is activated and one near miss, with streaming deltas, the MCP server listing, encrypted reasoning blobs and request ids removed and the local username replaced. Both pass the shared contract test, and the expected values were computed from the raw recordings rather than from the adapter. Not wired to the command line yet: there is no `--harness` flag, the preflight, credential check, model pin and judge still assume Claude, and the session's skills are not read back for the room check. Those follow. Tests: the command and environment, what the recording becomes (calls in order, a denied call's reason as its result, the answer, the tokens through the real usage recorder, a near miss), shapes the recordings lack (failed run, sub-agent message, malformed events), and the recording replayed by a real process through the engine's streaming loop, stopping at the skill call. Six deliberate breaks of the adapter each turn a test red. Signed-off-by: Eugene Volen --- docs/harnesses.md | 33 +- skillscope/harness/__init__.py | 2 + skillscope/harness/copilot.py | 193 ++++++++++++ tests/fixtures/harness/copilot/expected.json | 16 + .../fixtures/harness/copilot/no_trigger.jsonl | 43 +++ tests/fixtures/harness/copilot/trigger.jsonl | 45 +++ tests/test_copilot_harness.py | 294 ++++++++++++++++++ 7 files changed, 624 insertions(+), 2 deletions(-) create mode 100644 skillscope/harness/copilot.py create mode 100644 tests/fixtures/harness/copilot/expected.json create mode 100644 tests/fixtures/harness/copilot/no_trigger.jsonl create mode 100644 tests/fixtures/harness/copilot/trigger.jsonl create mode 100644 tests/test_copilot_harness.py diff --git a/docs/harnesses.md b/docs/harnesses.md index 67818cb..5f122bb 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -21,8 +21,37 @@ definition such as an `inspect_swe` one. Otherwise every agent would exist twice this repo, with its own command line, environment, skill staging and event handling, free to drift apart. -Today one harness is registered, `claude-code`. The contract exists so that a -second one is an adapter and a set of tests rather than a change to the engines. +Two harnesses are registered, `claude-code` and `copilot`, and the contract exists +so that another is an adapter and a set of tests rather than a change to the +engines. Neither can be chosen from the command line yet; see +[What is not yet behind the contract](#what-is-not-yet-behind-the-contract). + +## The harnesses + +| | `claude-code` | `copilot` | +| --- | --- | --- | +| Binary | `claude` | `copilot` | +| Skills directory | `.claude/skills` | `.claude/skills` | +| A skill activates by | a `Skill` tool call | a `skill` tool call | +| Session lists its skills | yes (`system/init`) | no; `copilot skill list --json` does, without a model call | +| Authenticates with | `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` | `COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or `GITHUB_TOKEN` | +| Config isolated by | `CLAUDE_CONFIG_DIR` | `COPILOT_HOME` | +| Prompt | stdin | stdin | +| A run is billed in | dollars | premium requests | + +Things to know about `copilot` before reading its numbers: + +* **Its room always contains built-in skills.** Copilot ships a few of its own, + so they are present beside the skills a run stages and can win a case. +* **It reports its answer separately from its closing event.** The harness + remembers the last assistant message and attaches it to the end of the run. +* **It reports tokens once, at the end,** and no dollar cost, so a spend cap in + dollars cannot bind. +* **It runs unsandboxed with every permission granted** (`--allow-all`), as the + host engine's Claude Code does, because a non-interactive run cannot answer a + prompt. Use a throwaway runner. +* **It uses its own model names** (`claude-sonnet-5.5`, not `sonnet`), and + models differ greatly in how many premium requests one run costs. ## What a harness provides diff --git a/skillscope/harness/__init__.py b/skillscope/harness/__init__.py index c641d5d..1487638 100644 --- a/skillscope/harness/__init__.py +++ b/skillscope/harness/__init__.py @@ -12,6 +12,7 @@ from .base import Harness, RunSpec from .claude_code import ClaudeCode +from .copilot import Copilot DEFAULT = "claude-code" @@ -42,5 +43,6 @@ def get(name: str | None = None) -> Harness: register(ClaudeCode()) +register(Copilot()) __all__ = ["DEFAULT", "Harness", "RunSpec", "get", "names", "register"] diff --git a/skillscope/harness/copilot.py b/skillscope/harness/copilot.py new file mode 100644 index 0000000..0ebfcf8 --- /dev/null +++ b/skillscope/harness/copilot.py @@ -0,0 +1,193 @@ +# Copyright Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: MIT + +"""GitHub Copilot CLI as a harness. + +`copilot --output-format json` prints one JSON event per line, none of them in +Claude Code's shape, so this translates. What the recorded output shows (CLI +1.0.105, `tests/fixtures/harness/copilot/`): + +* A tool call is `tool.execution_start` (`toolName`, `arguments`, + `toolCallId`) and its result is `tool.execution_complete`. Activating a skill + is a call to the tool named `skill` with `arguments.skill` set, which is the + call the engines already read. A call the permission rules deny is still + recorded, with `success: false`. +* The closing `result` event carries an exit code and premium-request usage and + **no answer**. The answer is the last assistant message, printed earlier, so + the translator remembers it. This is why a harness can keep state per run. +* Token counts arrive once, in `session.usage_checkpoint`, per model. +* There is no event naming the skills the session loaded. `copilot skill list + --json` does, without a model call, and is how the room can be checked. +* Streaming deltas, reasoning and bookkeeping events carry nothing the engines + read and are dropped. + +Other facts worth knowing before changing this: + +* It authenticates from `COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or `GITHUB_TOKEN` in + the environment, which is why an empty `COPILOT_HOME` is enough to isolate + the session from the runner's own configuration and user-level skills. +* Copilot ships built-in skills, so they are in every session's room beside the + ones staged here. +* It discovers skills in `.claude/skills`, the directory Claude Code uses. +* A run is billed in premium requests rather than dollars, so a spend cap in + dollars cannot bind. +""" + +from __future__ import annotations + +import os +from typing import Callable + +from .base import Harness, RunSpec + + +def _tool_use(call_id: str, name: str, arguments: dict) -> dict: + return { + "type": "assistant", + "message": { + "content": [ + {"type": "tool_use", "id": call_id, "name": name, "input": arguments} + ] + }, + } + + +def _tool_result(call_id: str, content: str) -> dict: + return { + "type": "user", + "message": { + "content": [ + {"type": "tool_result", "tool_use_id": call_id, "content": content} + ] + }, + } + + +def _tokens(data: dict) -> tuple[int, int]: + """Input and output tokens, summed over every model the run used.""" + metrics = ((data.get("accountingSnapshot") or {}).get("modelMetrics")) or {} + total_in = total_out = 0 + for per_model in metrics.values(): + usage = per_model.get("usage") if isinstance(per_model, dict) else None + if isinstance(usage, dict): + total_in += int(usage.get("inputTokens") or 0) + total_out += int(usage.get("outputTokens") or 0) + return total_in, total_out + + +class Copilot(Harness): + name = "copilot" + binary = "copilot" + skills_dir = ".claude/skills" + + def command(self, spec: RunSpec) -> list[str]: + # The prompt goes in on stdin, which this CLI reads when no `-p` is + # given: no quoting to get wrong, and no command-line length limit. + # + # `--allow-all` is Claude Code's `--dangerously-skip-permissions`: a + # non-interactive run cannot answer a prompt, so anything not + # pre-approved is denied and the agent could not do the work. Like the + # host engine's Claude Code, it runs unsandboxed. The built-in MCP + # servers are off so a case is graded on the skill, not on a tool the + # skill's author never had in mind. + cmd = [ + self.binary, + "--output-format", "json", + "--allow-all", + "--no-ask-user", + "--disable-builtin-mcps", + ] + if spec.model: + cmd += ["--model", spec.model] + if spec.effort: + cmd += ["--reasoning-effort", spec.effort] + cmd += list(spec.extra_flags) + return cmd + + def env(self, spec: RunSpec) -> dict[str, str]: + # A self-update in the middle of a graded run changes what is being + # graded, and the version is part of what a report is read against. The + # runner's own setting still wins, as it always did. The credential is + # the runner's to supply: this is only what the harness itself sets. + env = {"COPILOT_AUTO_UPDATE": os.environ.get("COPILOT_AUTO_UPDATE", "false")} + if spec.config_dir is not None: + env["COPILOT_HOME"] = spec.config_dir + return env + + def normalizer(self) -> Callable[[dict], list[dict]]: + # What the closing event needs and does not carry, remembered from the + # events before it. Per run, because the next case must not see it. + answer = {"text": ""} + tokens = {"input": 0, "output": 0} + + def normalize(event: dict) -> list[dict]: + kind = event.get("type") + data = event.get("data") + data = data if isinstance(data, dict) else {} + + if kind == "tool.execution_start": + arguments = data.get("arguments") + return [ + _tool_use( + str(data.get("toolCallId", "")), + str(data.get("toolName", "")), + arguments if isinstance(arguments, dict) else {}, + ) + ] + + if kind == "tool.execution_complete": + if data.get("success"): + content = (data.get("result") or {}).get("content") + else: + content = (data.get("error") or {}).get("message") + return [ + _tool_result( + str(data.get("toolCallId", "")), + content if isinstance(content, str) else "", + ) + ] + + if kind == "assistant.message": + # A sub-agent's message is its own, not the run's answer. + from_sub_agent = event.get("agentId") or data.get("agentId") + content = data.get("content") + if isinstance(content, str) and content.strip() and not from_sub_agent: + answer["text"] = content + return [] + + if kind == "session.usage_checkpoint": + tokens["input"], tokens["output"] = _tokens(data) + return [] + + if kind == "result": + failed = bool(event.get("exitCode")) + canonical: list[dict] = [] + if tokens["input"] or tokens["output"]: + # The usage recorder reads tokens off assistant events and + # this CLI reports them once, at the end. + canonical.append( + { + "type": "assistant", + "message": { + "content": [], + "usage": { + "input_tokens": tokens["input"], + "output_tokens": tokens["output"], + }, + }, + } + ) + canonical.append( + { + "type": "result", + "subtype": "error" if failed else "success", + "is_error": failed, + "result": answer["text"], + } + ) + return canonical + + return [] + + return normalize diff --git a/tests/fixtures/harness/copilot/expected.json b/tests/fixtures/harness/copilot/expected.json new file mode 100644 index 0000000..de13d6a --- /dev/null +++ b/tests/fixtures/harness/copilot/expected.json @@ -0,0 +1,16 @@ +{ + "note": "Recorded with GitHub Copilot CLI 1.0.105 on Windows. Streaming deltas, the MCP server listing, encrypted reasoning blobs and per-request ids were removed and the local username replaced; every other event is as printed. 'trigger' ran on claude-sonnet-5.5 and 'no_trigger' on claude-opus-5.5, shell and file writes denied.", + "skills": [ + "rocm-doctor" + ], + "trigger": { + "activation": "rocm-doctor", + "completed": true, + "final": "I couldn't confirm the cause on your machine. The `rocm` CLI the diagnosis depends on was blocked by a permission rule in this environment, so I ran nothing and changed nothing.\n\n**Most likely cause:** `hipErrorNoBinaryForGpu` matches the catalog entry `fix-1-arch`. Your GPU's `gfx` target isn't in the build list of your PyTorch wheel or ROCm build, so no compiled kernels exist for it. That also explains `torch.cuda.is_available()` returning `False`. I haven't verified this against your hardware.\n\n**Next steps:**\n1. Run `rocm diagnose --symptom \"hipErrorNoBinaryForGpu\" --json` yourself. Check that `has_match` is true and look at the `fix` section.\n2. If `rocm` isn't installed, get it from https://github.com/ROCm/rocm-cli.\n3. For `fix-1-arch`, `rocm fix fix-1-arch` only prints a plan, and you apply it yourself. It won't change anything for you.\n4. Don't set `HSA_OVERRIDE_GFX_VERSION` by hand. It can cause page faults.\n\nIf you'd rather I run the commands, allow shell access and tell me your OS and GPU model." + }, + "no_trigger": { + "activation": null, + "completed": true, + "final": "I couldn't diagnose your machine because shell access and file access outside this workspace were both denied, so nothing has been changed yet. The ROCm troubleshooting skill doesn't apply here because it only covers AMD GPUs.\n\nRun these in the Python environment you use:\n\n1. **Check the driver:** `nvidia-smi`. If it fails, reinstall the latest NVIDIA Game Ready or Studio driver and reboot.\n2. **Check your PyTorch build:** `python -c \"import torch; print(torch.__version__, torch.version.cuda)\"`. If it shows `+cpu` or `None`, you have the CPU-only build. This is the most common cause.\n3. **Fix a CPU-only build:**\n `pip uninstall -y torch torchvision torchaudio`\n `pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124`\n4. **Check the environment:** make sure `CUDA_VISIBLE_DEVICES` isn't set to `-1` or left empty.\n\nYou don't need to install the CUDA Toolkit separately, because the PyTorch wheels include it. Send me the output of steps 1 and 2, or allow shell access, and I'll fix it." + } +} diff --git a/tests/fixtures/harness/copilot/no_trigger.jsonl b/tests/fixtures/harness/copilot/no_trigger.jsonl new file mode 100644 index 0000000..376ec51 --- /dev/null +++ b/tests/fixtures/harness/copilot/no_trigger.jsonl @@ -0,0 +1,43 @@ +{"type": "session.tools_updated", "data": {"model": "claude-opus-5.5"}, "ephemeral": true, "id": "f13334b8-16dd-41f4-aa27-ab47e80bd8ae", "timestamp": "2026-10-29T10:26:36.917Z", "parentId": "e5841a81-d879-43bd-9dab-d1f5e833b635"} +{"type": "user.message", "data": {"content": "torch.cuda.is_available() is False on my NVIDIA RTX 4090 and CUDA seems broken. Help me fix it.", "transformedContent": "2026-10-29T12:26:36.920+02:00\n\ntorch.cuda.is_available() is False on my NVIDIA RTX 4090 and CUDA seems broken. Help me fix it.", "messageId": "b5b03a67-59f4-4ff4-83c9-1d75ae27dc18", "supportedNativeDocumentMimeTypes": [], "delivery": "idle", "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b", "turnId": "0", "parentAgentTaskId": "8d3fc53b-85f0-49c6-958f-e59130dd8616", "responsesReasoning": {"model": "claude-opus-5.5", "initialEffort": "high", "effort": "high"}}, "id": "1c87385a-e895-49ab-88ff-5323b9e1e1a7", "timestamp": "2026-10-29T10:26:36.920Z", "parentId": "e5841a81-d879-43bd-9dab-d1f5e833b635"} +{"type": "assistant.turn_start", "data": {"turnId": "0", "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b"}, "id": "7ca84d0a-325b-417f-a9d9-55e92a29abce", "timestamp": "2026-10-29T10:26:36.937Z", "parentId": "c0189526-75d9-4a46-a113-23dce4781bdd"} +{"type": "model.call_start", "data": {"turnId": "0", "model": "claude-opus-5.5"}, "ephemeral": true, "id": "88c5128c-674b-498f-8edb-eb21fec909f2", "timestamp": "2026-10-29T10:26:36.955Z", "parentId": "7ca84d0a-325b-417f-a9d9-55e92a29abce"} +{"type": "assistant.message_start", "data": {"messageId": "d9c233a5-0006-403c-ba14-822d173dd511"}, "ephemeral": true, "id": "386ee612-42bf-4d4a-9d48-b318ddd72e0b", "timestamp": "2026-10-29T10:26:39.747Z", "parentId": "7ca84d0a-325b-417f-a9d9-55e92a29abce"} +{"type": "model.call_finished", "data": {"turnId": "0", "dispatchDurationMs": 4002, "outcome": "success", "editClassifierVersion": 1, "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b", "containsBuiltInFileEditRequest": false}, "ephemeral": true, "id": "fd9ba09a-1aef-4d7f-be03-c796e93751c8", "timestamp": "2026-10-29T10:26:40.969Z", "parentId": "4a5fd640-892e-462e-9dc0-832c18feb4e2"} +{"type": "assistant.message", "data": {"messageId": "d9c233a5-0006-403c-ba14-822d173dd511", "originatingMessageId": "b5b03a67-59f4-4ff4-83c9-1d75ae27dc18", "model": "claude-opus-5.5", "providerId": "copilot", "content": "This is an NVIDIA setup, so the ROCm skill doesn't apply. I'll start by checking the driver, Python, and PyTorch build.", "toolRequests": [{"toolCallId": "toolu_01W9tMLdyAxNw7dbd9T43wZf", "name": "powershell", "arguments": {"command": "nvidia-smi 2>&1 | Select-Object -First 20; \"---\"; Get-Command python, py, nvcc -ErrorAction SilentlyContinue | Format-Table Name, Source -AutoSize; \"---\"; python -c \"import sys,torch;print(sys.executable);print(torch.__version__, torch.version.cuda, torch.cuda.is_available())\" 2>&1", "description": "Diagnose NVIDIA driver and torch"}, "type": "function", "intentionSummary": "Diagnose NVIDIA driver and torch", "toolTitle": "Running command"}], "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b", "turnId": "0", "reasoningText": "This user has an NVIDIA GPU rather than AMD, so the rocm-doctor skill doesn't apply here—I'll skip it and diagnose directly by checking nvidia-smi output and the torch version instead.", "rte": true, "apiCallId": "msg_011CgWNKCQVq4SGgYvbraH29", "reasoningBlocks": {"provider": "anthropic-messages", "blocks": [{"signature": "CAQS0wYKEAgTGAI4AUIIdGhpbmtpbmcSDAOfuMzBSbDZFj80jBoMBGRc8//UNviTg0ueIjAjPnI+g7VghzzIH3f4FcWRoxPNd0VpV/Z5iHB2/3c3MovpZjGRcaj92t2dCYWA6cUq8AV4J2hme1nnVVpxk/pr1YrCjxU4fbiJXXstmQkz3FgcpK1QKa97I00OupACaowtd3Ts5X4aBRw96Xubf7rFpXBGn9VzEa/u/Bvl/FZHoF/GqyWjwl0Qwzp/HqN7I5mMqngEdiu2XMpTHVCYu71m8IPIGCCuBai+toG5mPbbZSkURt2bWrIBjgmf9TFS8WUzSOdMd/z1Lz1pmaAcy8zWrkCNOscish+AX1JYT9FxO1cQDqCrC9d74kofIxY614dBqEiFz/LxyWIo88/JDYF5Z5hjelY66p+XHf6U10dGDQ1qWYh+6gejDFdMdkpVEVGc5tzkL+VoW0yzrWKn4lvsX2oIkus9opOSllrJRusvhqPfyA56GKs+cI7o4Z7XJMe9GTtdZCsN3f0SJNCPeadfATOc7G7+zLMCE/YrjnzDR3O95uGBxCeWPQEZIuqGcrDxv8bVC5/fsvfyDDDGi++lzyeK2FKSUyzOHRJ8sYDuaev/SU6IaLnGQCveUeZArSnoz1rOIhcugYXM+MdD0RauSZvms9sRihC0DjqJ3c4Hu8xXVFSB9X/Y17/pFuowAriBjsGQ49gDOlDqhWwzUcTlyF2SbplTIDi3xqU+tFJRDEDzH3ayePDHPbwOLno+/bUf0wSP9mSFPiT8dJYQmEJM3XF9ude2AU+i9rmbve/bdz5yXYZkzQ4bIA5khsHhnsIuhZMdbDOOBwERio4oz/pEwKg3d1Cp57cYmQbDEl/sxnn+/oAvtXerckAw+4pRaGWpvVG/xAXtw84GkqQiw8/pe8+4+WJMAMaBIasqzYv1b0sesJXkO4SS98ytsNnohtRVXHeAu1i4SsASI2EwRKmj0H1a8NKx+VRsbH7ceN8kguE2k/UA43Nv9PZXbqvDx+G4ksA5g/AKwNKanoDIp7magz25qe58Ai8P/KCSikCvIdIJwafgciu27cb2cCha71OuDqMsiHccgvOHwTXByKfzeG8+VtlE6VL+XayJTAW9s16zJRgB", "thinking": "This user has an NVIDIA GPU rather than AMD, so the rocm-doctor skill doesn't apply here—I'll skip it and diagnose directly by checking nvidia-smi output and the torch version instead.\n\n", "type": "thinking"}]}}, "id": "c5d45eda-c755-40b5-93ca-caab29868284", "timestamp": "2026-10-29T10:26:40.972Z", "parentId": "4a5fd640-892e-462e-9dc0-832c18feb4e2"} +{"type": "assistant.reasoning", "data": {"reasoningId": "97494893-fe9e-4c96-a879-2096cd937735", "content": "This user has an NVIDIA GPU rather than AMD, so the rocm-doctor skill doesn't apply here—I'll skip it and diagnose directly by checking nvidia-smi output and the torch version instead.", "rte": true}, "ephemeral": true, "id": "1407472e-f189-4c48-b145-a636995b84aa", "timestamp": "2026-10-29T10:26:40.974Z", "parentId": "c5d45eda-c755-40b5-93ca-caab29868284"} +{"type": "tool.execution_start", "data": {"toolCallId": "toolu_01W9tMLdyAxNw7dbd9T43wZf", "toolName": "powershell", "arguments": {"command": "nvidia-smi 2>&1 | Select-Object -First 20; \"---\"; Get-Command python, py, nvcc -ErrorAction SilentlyContinue | Format-Table Name, Source -AutoSize; \"---\"; python -c \"import sys,torch;print(sys.executable);print(torch.__version__, torch.version.cuda, torch.cuda.is_available())\" 2>&1", "description": "Diagnose NVIDIA driver and torch"}, "turnId": "0", "model": "claude-opus-5.5", "toolTitle": "Running command", "shellToolInfo": {"possiblePaths": ["Source", "python", "Name"], "hasWriteFileRedirection": true}}, "id": "084b2112-349e-48f7-8e27-2e2ab5c8f58d", "timestamp": "2026-10-29T10:26:40.977Z", "parentId": "c5d45eda-c755-40b5-93ca-caab29868284"} +{"type": "model.call_final_result", "data": {"model": "claude-opus-5.5", "isByok": false, "result": "success"}, "ephemeral": true, "id": "a41ac604-a070-4913-b81c-e3e5c916dd73", "timestamp": "2026-10-29T10:26:40.980Z", "parentId": "084b2112-349e-48f7-8e27-2e2ab5c8f58d"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "3e35a804-45c3-4cbe-8fd8-ef8c52bcb2fc", "timestamp": "2026-10-29T10:26:41.008Z", "parentId": "084b2112-349e-48f7-8e27-2e2ab5c8f58d"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "96a139fc-abca-455a-8939-bed0e383410a", "timestamp": "2026-10-29T10:26:41.009Z", "parentId": "084b2112-349e-48f7-8e27-2e2ab5c8f58d"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "272d1fcf-5dec-4e06-b7da-803a78caf00e", "timestamp": "2026-10-29T10:26:41.009Z", "parentId": "084b2112-349e-48f7-8e27-2e2ab5c8f58d"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "a4e56929-dd22-4d90-b763-873cbbedc135", "timestamp": "2026-10-29T10:26:41.009Z", "parentId": "084b2112-349e-48f7-8e27-2e2ab5c8f58d"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "9f3d2229-be81-44fc-a438-e651e3bd4457", "timestamp": "2026-10-29T10:26:41.010Z", "parentId": "084b2112-349e-48f7-8e27-2e2ab5c8f58d"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "1762021b-6a58-4137-afb6-3674d675e30c", "timestamp": "2026-10-29T10:26:41.010Z", "parentId": "084b2112-349e-48f7-8e27-2e2ab5c8f58d"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "77cd7162-0051-463d-acd1-3d432bb1e406", "timestamp": "2026-10-29T10:26:41.015Z", "parentId": "084b2112-349e-48f7-8e27-2e2ab5c8f58d"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "5d95fd8e-1038-463e-91ad-9a303152a151", "timestamp": "2026-10-29T10:26:41.016Z", "parentId": "084b2112-349e-48f7-8e27-2e2ab5c8f58d"} +{"type": "tool.execution_complete", "data": {"toolCallId": "toolu_01W9tMLdyAxNw7dbd9T43wZf", "model": "claude-opus-5.5", "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b", "turnId": "0", "rte": true, "success": false, "error": {"message": "Permission to run this tool was denied due to the following rules: `shell`", "code": "denied"}, "toolTelemetry": {"properties": {"shell_error_category": "permission_denied"}, "metrics": {"context_token_limit": 200000, "context_current_tokens": 17288, "large_output_threshold_bytes": 20480}}}, "id": "6897a88c-254b-4b99-b241-95f382281118", "timestamp": "2026-10-29T10:26:41.017Z", "parentId": "084b2112-349e-48f7-8e27-2e2ab5c8f58d"} +{"type": "assistant.turn_end", "data": {"turnId": "0"}, "id": "d34ac652-24bc-46c5-b5e5-02ce56e647cc", "timestamp": "2026-10-29T10:26:41.022Z", "parentId": "6897a88c-254b-4b99-b241-95f382281118"} +{"type": "assistant.turn_start", "data": {"turnId": "1", "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b"}, "id": "ad95079f-17a1-45b2-b35a-11774366943c", "timestamp": "2026-10-29T10:26:41.023Z", "parentId": "d34ac652-24bc-46c5-b5e5-02ce56e647cc"} +{"type": "model.call_start", "data": {"turnId": "1", "model": "claude-opus-5.5"}, "ephemeral": true, "id": "aa8c63d0-b203-408c-95e3-3a4f440d9fe5", "timestamp": "2026-10-29T10:26:41.033Z", "parentId": "ad95079f-17a1-45b2-b35a-11774366943c"} +{"type": "assistant.message_start", "data": {"messageId": "bfb28236-4e7e-4934-88f9-5a670baef048"}, "ephemeral": true, "id": "05f45210-85d3-4e76-b152-5159e4c9f8c0", "timestamp": "2026-10-29T10:26:44.815Z", "parentId": "ad95079f-17a1-45b2-b35a-11774366943c"} +{"type": "model.call_finished", "data": {"turnId": "1", "dispatchDurationMs": 4306, "outcome": "success", "editClassifierVersion": 1, "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b", "containsBuiltInFileEditRequest": false}, "ephemeral": true, "id": "47856964-f355-45ab-b92f-9ffae74ad8cc", "timestamp": "2026-10-29T10:26:45.346Z", "parentId": "e89a04b8-e7b5-4aa0-9112-33df092f8624"} +{"type": "assistant.message", "data": {"messageId": "bfb28236-4e7e-4934-88f9-5a670baef048", "originatingMessageId": "b5b03a67-59f4-4ff4-83c9-1d75ae27dc18", "model": "claude-opus-5.5", "providerId": "copilot", "content": "Shell access is denied, so I can't run commands. I'll check the installed files directly instead.", "toolRequests": [{"toolCallId": "toolu_01AhaFznQ1Gxs7pVubm512Dy", "name": "glob", "arguments": {"pattern": "**/torch-*.dist-info/METADATA", "paths": ["C:\\Users\\user\\AppData\\Local\\Programs\\Python", "C:\\Users\\user\\AppData\\Roaming\\Python"]}, "type": "function", "intentionSummary": "**/torch-*.dist-info/METADATA", "toolTitle": "Finding files"}, {"toolCallId": "toolu_01MWaQwdP1fpBF9wcE4z3N3x", "name": "glob", "arguments": {"pattern": "nvidia-smi.exe", "paths": ["C:\\Windows\\System32"]}, "type": "function", "intentionSummary": "nvidia-smi.exe", "toolTitle": "Finding files"}], "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b", "turnId": "1", "reasoningText": "Since shell commands are denied, I'm considering other tools like glob or view to check installed torch versions, perhaps by peeking into site-packages or likely installation directories, even though searching outside the working directory should only be done when truly necessary.", "rte": true, "apiCallId": "msg_011CgWNKV4s3PkdH2y7hnnBy", "reasoningBlocks": {"provider": "anthropic-messages", "blocks": [{"signature": "CAQS8QgKEAgTGAI4AUIIdGhpbmtpbmcSDHuHrj+ce3i6u592DhoMA6eoDAnbLI9zDelEIjDijmP/RzH2wFJGsEjzj5IUkRqUhu3nbY2qPSjJdxVoCsk84BEqWDmBFrUr2Ks3LHQqjgiz8+u/cg3u8GZaq6frEQXJzFQUSspSLwIqP4dH13woZIcYKKd5kcxaiu1Zac3vLPKwzIm0yKTzN8figoDYCh+zrmREb13GHhYdR8InNpHS+BkVjbsmgSx5bNVJNu2fiFheIjWPGwwT10ouoebw7O3qg+3UHnLAZiefuqkzHvjZAvK0H9Wq9i7W3MW+cQnPeIXkkrZ3nF7ZtaWPSEAJcLClrfaXPXs8JG9ATfXjWw57UCOvluE/Cxxby9MLfQXgLecY90P2bYFkyCp3ue/jpyIYNbhTO4s7hcos7uclhr/5mMy435NnxyXRc5xPLT5saQe699XH8EGNkoXnW+IBV1r05GZ1q55cF29meRdV2/y3248kFuhyezoPfhAq4aB7/AOqnyoKsbTPn9EcfEKkNbHKEKTc9Pl7VAQK6NEvp4rWKknGymOkmzM24avjMorg69Mu1LPHOcDy0boJczwYUJXuqaJRAqOtYYPs4gTxMUH/UrFC4doi0lcIi70xEKSMs2VQPhx1w3vd7U71KiKgn3+a0FZaNn/6Ph/fhQwpFofWpQ2HV00MTFUMZCRdLOzzwl+9Tzzd4f1Hr/xO3l8VZznMbQQ7Tqn+mxyTawfB3EhaWXypVBOJFjaG+ARaFtx71NRZOBFXXMOcPis6Xhezn/jZwFYhEpCr1/3h7Cm+TW/N+n6Vp+m7cjba1klQMVJbT4DUmpk6tg/tlzaDmZeNb/zx917RDNGY6XOcRmFSS05EnYMVbHLzgNxzUbAxsJnW1pWnJm8foNgxEkL79cO/yQulcgtn2MNSvt9TbJYOXxhHez6VZzq6yHJZiJpR8LFF0khImzZHszL2jlEJhZOIsM6VnSK4Up/5IPmK1Pw8H3xzQ+bmx6W4akkC9OS2UJQCA7UgqCwSlParLlJ8tFGkjfAf8hctAvZjUAGCqProsjcehH5O3GsHGiVemVfWDiC5WjHb4O9lB/Rm9yrOIRURkQ2RVYUQjNK4ntRfSx55Y5PcNPUdQG+CPASh9NdzpwMVv8m3Y1NB/eKISGlvTabpdrF+qt5McQ4TlDn6YFdgTAqIfNNxkaRZLud81wnGpLYEy6pbUlUsQ+nEiA5ogGxaC3sRr6Z5XTeG/QDanVAW/lVubF43lMvN1s2wS+I75czFGczj81qviuTP4+F8mfZurxW/X40LdDQl0tMP+sxs/RhMi08JI8GG/ncoB1R12mHntgwnlyD7t+7tLSXnnEjJPzmkRmPk8xW1Fbd2Jx29HyCN4JluzrhIeGKA5FlkShqxiYlTdOvC87qTX4wPHMtregboU5FV3uPwJ3Bny948EcZDlTQrrWZGWhH4IxT/kcGhQwQKdjtjVcy6pajjAKgM6dKovTJcPdeBwqwkqk9aaeEYAQ==", "thinking": "Since shell commands are denied, I'm considering other tools like glob or view to check installed torch versions, perhaps by peeking into site-packages or likely installation directories, even though searching outside the working directory should only be done when truly necessary.\n\n", "type": "thinking"}]}}, "id": "c3c8a68b-d6d2-4f62-82ee-73020750026c", "timestamp": "2026-10-29T10:26:45.348Z", "parentId": "e89a04b8-e7b5-4aa0-9112-33df092f8624"} +{"type": "assistant.reasoning", "data": {"reasoningId": "e2db0d53-4ee2-41b1-8fd0-db26f1e84355", "content": "Since shell commands are denied, I'm considering other tools like glob or view to check installed torch versions, perhaps by peeking into site-packages or likely installation directories, even though searching outside the working directory should only be done when truly necessary.", "rte": true}, "ephemeral": true, "id": "2b3d3086-c7db-474e-9ac4-3d53379c8670", "timestamp": "2026-10-29T10:26:45.349Z", "parentId": "c3c8a68b-d6d2-4f62-82ee-73020750026c"} +{"type": "tool.execution_start", "data": {"toolCallId": "toolu_01AhaFznQ1Gxs7pVubm512Dy", "toolName": "glob", "arguments": {"pattern": "**/torch-*.dist-info/METADATA", "paths": ["C:\\Users\\user\\AppData\\Local\\Programs\\Python", "C:\\Users\\user\\AppData\\Roaming\\Python"]}, "turnId": "1", "model": "claude-opus-5.5", "toolTitle": "Finding files"}, "id": "5c0c3db1-8484-4ee9-9e16-c89966f18c98", "timestamp": "2026-10-29T10:26:45.351Z", "parentId": "c3c8a68b-d6d2-4f62-82ee-73020750026c"} +{"type": "tool.execution_start", "data": {"toolCallId": "toolu_01MWaQwdP1fpBF9wcE4z3N3x", "toolName": "glob", "arguments": {"pattern": "nvidia-smi.exe", "paths": ["C:\\Windows\\System32"]}, "turnId": "1", "model": "claude-opus-5.5", "toolTitle": "Finding files"}, "id": "ec128528-8414-4497-9945-9af7e3c84896", "timestamp": "2026-10-29T10:26:45.353Z", "parentId": "5c0c3db1-8484-4ee9-9e16-c89966f18c98"} +{"type": "model.call_final_result", "data": {"model": "claude-opus-5.5", "isByok": false, "result": "success"}, "ephemeral": true, "id": "5036e942-7d56-4c93-83d1-4d0efd7c0e61", "timestamp": "2026-10-29T10:26:45.355Z", "parentId": "ec128528-8414-4497-9945-9af7e3c84896"} +{"type": "tool.execution_complete", "data": {"toolCallId": "toolu_01AhaFznQ1Gxs7pVubm512Dy", "model": "claude-opus-5.5", "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b", "turnId": "1", "rte": true, "success": false, "error": {"message": "Permission denied because no interactive user response was available. Retry in an interactive session so the user can approve it, or try an alternative that does not require this permission.", "code": "denied"}, "toolTelemetry": {"properties": {}, "metrics": {"context_token_limit": 200000, "context_current_tokens": 17698, "large_output_threshold_bytes": 20480}}}, "id": "181d6cb1-969c-43c0-88c4-52ea6676c86c", "timestamp": "2026-10-29T10:26:45.368Z", "parentId": "ecc55e2a-8e83-46ca-a266-4e82894e1b8e"} +{"type": "tool.execution_complete", "data": {"toolCallId": "toolu_01MWaQwdP1fpBF9wcE4z3N3x", "model": "claude-opus-5.5", "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b", "turnId": "1", "rte": true, "success": false, "error": {"message": "Permission denied because no interactive user response was available. Retry in an interactive session so the user can approve it, or try an alternative that does not require this permission.", "code": "denied"}, "toolTelemetry": {"properties": {}, "metrics": {"context_token_limit": 200000, "context_current_tokens": 17698, "large_output_threshold_bytes": 20480}}}, "id": "624f27ba-295e-4995-82ff-83e4c5ce43e7", "timestamp": "2026-10-29T10:26:45.373Z", "parentId": "181d6cb1-969c-43c0-88c4-52ea6676c86c"} +{"type": "assistant.turn_end", "data": {"turnId": "1"}, "id": "afa096dd-c695-4db7-8c1c-b32491f66a94", "timestamp": "2026-10-29T10:26:45.378Z", "parentId": "624f27ba-295e-4995-82ff-83e4c5ce43e7"} +{"type": "assistant.turn_start", "data": {"turnId": "2", "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b"}, "id": "86d46b8c-df18-40d8-9d13-a47b8cb1f969", "timestamp": "2026-10-29T10:26:45.379Z", "parentId": "afa096dd-c695-4db7-8c1c-b32491f66a94"} +{"type": "model.call_start", "data": {"turnId": "2", "model": "claude-opus-5.5"}, "ephemeral": true, "id": "b7b9de51-bd2c-414f-854c-85ca878c1f0d", "timestamp": "2026-10-29T10:26:45.389Z", "parentId": "86d46b8c-df18-40d8-9d13-a47b8cb1f969"} +{"type": "assistant.message_start", "data": {"messageId": "22a99d1a-19f6-47a9-a6c1-6b33f4140dac"}, "ephemeral": true, "id": "294a7756-6136-4b97-bfdc-9041cb69054e", "timestamp": "2026-10-29T10:26:50.319Z", "parentId": "86d46b8c-df18-40d8-9d13-a47b8cb1f969"} +{"type": "model.call_finished", "data": {"turnId": "2", "dispatchDurationMs": 5055, "outcome": "success", "editClassifierVersion": 1, "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b", "containsBuiltInFileEditRequest": false}, "ephemeral": true, "id": "988f6e77-cd00-480f-ac0c-b4b081ec4324", "timestamp": "2026-10-29T10:26:50.452Z", "parentId": "12fde6a1-7b31-4976-82c4-c01c72f3c0d2"} +{"type": "assistant.message", "data": {"messageId": "22a99d1a-19f6-47a9-a6c1-6b33f4140dac", "originatingMessageId": "b5b03a67-59f4-4ff4-83c9-1d75ae27dc18", "model": "claude-opus-5.5", "providerId": "copilot", "content": "I couldn't diagnose your machine because shell access and file access outside this workspace were both denied, so nothing has been changed yet. The ROCm troubleshooting skill doesn't apply here because it only covers AMD GPUs.\n\nRun these in the Python environment you use:\n\n1. **Check the driver:** `nvidia-smi`. If it fails, reinstall the latest NVIDIA Game Ready or Studio driver and reboot.\n2. **Check your PyTorch build:** `python -c \"import torch; print(torch.__version__, torch.version.cuda)\"`. If it shows `+cpu` or `None`, you have the CPU-only build. This is the most common cause.\n3. **Fix a CPU-only build:**\n `pip uninstall -y torch torchvision torchaudio`\n `pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124`\n4. **Check the environment:** make sure `CUDA_VISIBLE_DEVICES` isn't set to `-1` or left empty.\n\nYou don't need to install the CUDA Toolkit separately, because the PyTorch wheels include it. Send me the output of steps 1 and 2, or allow shell access, and I'll fix it.", "toolRequests": [], "interactionId": "1c1034a2-beb8-4572-b990-a73cbb854f0b", "turnId": "2", "reasoningText": "I can't actually inspect the system directly here, so I should give concise guidance instead, noting that the ROCm skill doesn't apply, and outline practical steps the user can take.", "rte": true, "apiCallId": "msg_011CgWNKoPSETpixsddEL794", "reasoningBlocks": {"provider": "anthropic-messages", "blocks": [{"signature": "CAQSxAYKEAgTGAI4AUIIdGhpbmtpbmcSDLh0mAPWcqrrtpf52hoMWSkBuLsm0pc2gg/sIjBdGupB/ZjQN5mretT4RnmNxZzyonQ3kpBX0S/pSPkIhHjdvlT9UI+qwpAVmq99M+cq4QU8xBevDxhp1MqjQiAoJnchoQm22mg/syc5iOzNKx13SF3XUKIQUE92De7AHJiJVzT0ZuNqLyhYLmjCXGKtqcpLcoqqpeK9NB0fSaxKL59ZM0zxJwg1XKQvA8lRSzp0PuZ26Z4V7GJMXl7wwWzNYqYCAdHHH/wFgXfCDePkhTq1qQodNWFjHWBHVglHUzCuzYo2Zvqmq0VIAqqMnWncwJ2EHHBTESVziAvkgjX1IcWYtLRJ42W9XN++OOcQVhp0ytVDvwyjLVJzgxg+dodZfuSA6Bk2a+C2XMR0yvwnB7ro5Hk/CTIs8xCUxoe25XPEmsFfGkjpJglylwFYa0wM1yCndERywrT4EeFycS+7kd1rsU97LIvNAWj25uZJfrOLP0qybtDve7YWPwm2fjOQg5HHrqUKwFQiXO4UuedXbKZDauZnqAP8CTMKDfGQmIkkoekIzL/jagNgWTL/61EKV67HhhJ4FtbVTh4jxv6DFdwXNfVbpVr4+8RC6vnJ8osVf3enzHFz66buCthUfc6ISkzzggAhChuCdExncBnAqq8fiDOp72KNfyTVmS/iyKeJf/hHd3XhDgdS0GocDhVjgbdHmFPkY+a8jFj2B2VSy6tLU3CZXXqj0Tp240rMo9xqKm9/EqHOjRirX5tNfw04XwJhSoT0lWI+eGwhH4Rpr1+lwxTWbbTyOh+j+lKbfQ/+8S7BHYrYSPzc2tJVC1AZ8u+0yeXbKcM66Kxudugx7bEqpj6mt1Fwa3TRifnZ49SoOEZAEsH/0GYBjA3fbl6LByHyO84ABE+btD9sEsTJ6ysrEbAYutOlRsnUgsQhwMeM1P/Iixj0oPI9kQWXxbmmTYw3bz4OrZ6Yk0F7gmq7tXBTUQyy0+CeBgQtwDoCmyHNVZPzT34Uo5chmtz3Td3JT4QLuvrbQjaddTi+MOSh0AMxZTt12+Hjjsk1bemU40SbVA3vPMGZMwpWDVA1xPrie+ER3RgB", "thinking": "I can't actually inspect the system directly here, so I should give concise guidance instead, noting that the ROCm skill doesn't apply, and outline practical steps the user can take.\n\n", "type": "thinking"}]}}, "id": "9e78963b-a86d-41fd-a50c-4673d561b294", "timestamp": "2026-10-29T10:26:50.455Z", "parentId": "12fde6a1-7b31-4976-82c4-c01c72f3c0d2"} +{"type": "assistant.reasoning", "data": {"reasoningId": "5077e1cc-b760-482a-a3a0-fd5aedb351ee", "content": "I can't actually inspect the system directly here, so I should give concise guidance instead, noting that the ROCm skill doesn't apply, and outline practical steps the user can take.", "rte": true}, "ephemeral": true, "id": "cc49b60e-bcb2-4071-8c0d-cd0013953b7f", "timestamp": "2026-10-29T10:26:50.456Z", "parentId": "9e78963b-a86d-41fd-a50c-4673d561b294"} +{"type": "model.call_final_result", "data": {"model": "claude-opus-5.5", "isByok": false, "result": "success"}, "ephemeral": true, "id": "e632d22a-cec4-466e-8649-0bfb3e246924", "timestamp": "2026-10-29T10:26:50.458Z", "parentId": "9e78963b-a86d-41fd-a50c-4673d561b294"} +{"type": "assistant.turn_end", "data": {"turnId": "2"}, "id": "ee9556b0-578a-475f-97e8-71eac0a32cd9", "timestamp": "2026-10-29T10:26:50.458Z", "parentId": "9e78963b-a86d-41fd-a50c-4673d561b294"} +{"type": "session.usage_checkpoint", "data": {"totalNanoAiu": 15622680000, "totalPremiumRequests": 15, "modelCacheState": [{"modelId": "claude-opus-5.5", "cacheExpiresAt": "2026-10-29T10:31:45.391Z", "cacheTtlSeconds": 300}], "accountingSnapshot": {"totalPremiumRequestCost": 15, "totalUserRequests": 1, "totalNanoAiu": 15622680000, "aiCreditsStatus": "complete", "tokenDetails": {"input": {"tokenCount": 8}, "cache_read": {"tokenCount": 47399}, "cache_write": {"tokenCount": 24455}, "output": {"tokenCount": 1222}}, "totalApiDurationMs": 13358, "sessionStartTime": "2026-10-29T10:26:35.123Z", "codeChanges": {"linesAdded": 0, "linesRemoved": 0, "filesModifiedCount": 0, "filesModified": []}, "modelMetrics": {"claude-opus-5.5": {"requests": {"count": 3, "cost": 15}, "usage": {"inputTokens": 71862, "outputTokens": 1222, "cacheReadTokens": 47399, "cacheWriteTokens": 24455, "reasoningTokens": 281}, "cacheExpiresAt": "2026-10-29T10:31:45.391Z", "totalNanoAiu": 15622680000, "aiCreditsStatus": "complete", "tokenDetails": {"input": {"tokenCount": 8}, "cache_read": {"tokenCount": 47399}, "cache_write": {"tokenCount": 24455}, "output": {"tokenCount": 1222}}}}, "agentMetrics": {"main": {"totalApiDurationMs": 13358, "totalNanoAiu": 15622680000, "modelMetrics": {"claude-opus-5.5": {"requests": {"count": 3, "cost": 15}, "usage": {"inputTokens": 71862, "outputTokens": 1222, "cacheReadTokens": 47399, "cacheWriteTokens": 24455, "reasoningTokens": 281}, "totalNanoAiu": 15622680000, "tokenDetails": {"input": {"tokenCount": 8}, "cache_read": {"tokenCount": 47399}, "cache_write": {"tokenCount": 24455}, "output": {"tokenCount": 1222}}}}}}, "currentModel": "claude-opus-5.5", "lastCallInputTokens": 24457, "lastCallOutputTokens": 448}, "usageAccountingWatermarks": {"3cde4095-9e95-41fe-94c7-307bd52ffc98": 3}}, "id": "9adb3473-24c2-4c4a-87b6-33f4393f5a5d", "timestamp": "2026-10-29T10:26:50.463Z", "parentId": "ee9556b0-578a-475f-97e8-71eac0a32cd9"} +{"type": "assistant.idle", "data": {}, "ephemeral": true, "id": "88a65ee6-bd6a-45a9-b668-1030894e8fa3", "timestamp": "2026-10-29T10:26:50.464Z", "parentId": "9adb3473-24c2-4c4a-87b6-33f4393f5a5d"} +{"type": "result", "timestamp": "2026-10-29T10:26:50.479Z", "sessionId": "3cde4095-9e95-41fe-94c7-307bd52ffc98", "exitCode": 0, "usage": {"premiumRequests": 15, "totalApiDurationMs": 13358, "sessionDurationMs": 15356, "codeChanges": {"linesAdded": 0, "linesRemoved": 0, "filesModified": []}}} diff --git a/tests/fixtures/harness/copilot/trigger.jsonl b/tests/fixtures/harness/copilot/trigger.jsonl new file mode 100644 index 0000000..d8c1449 --- /dev/null +++ b/tests/fixtures/harness/copilot/trigger.jsonl @@ -0,0 +1,45 @@ +{"type": "session.tools_updated", "data": {"model": "claude-sonnet-5.5"}, "ephemeral": true, "id": "9e38fbb2-1b96-4dbb-a30f-d8c586288f0e", "timestamp": "2026-10-29T10:25:24.335Z", "parentId": "169cfe8e-7cda-4b7b-8702-9334840c8bea"} +{"type": "user.message", "data": {"content": "torch.cuda.is_available() returns False on my AMD GPU and I get 'hipErrorNoBinaryForGpu' when I run my script. What's wrong?", "transformedContent": "2026-10-29T12:25:24.336+02:00\n\ntorch.cuda.is_available() returns False on my AMD GPU and I get 'hipErrorNoBinaryForGpu' when I run my script. What's wrong?", "messageId": "4e46f416-a8fd-4fa6-a881-2e5d6ecfd5ce", "supportedNativeDocumentMimeTypes": [], "delivery": "idle", "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "turnId": "0", "parentAgentTaskId": "a8ef17f5-43d5-401c-a3a3-3d82a391b574"}, "id": "de8a5e09-9203-473c-9a52-6fadb1f9dcb4", "timestamp": "2026-10-29T10:25:24.337Z", "parentId": "169cfe8e-7cda-4b7b-8702-9334840c8bea"} +{"type": "assistant.turn_start", "data": {"turnId": "0", "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e"}, "id": "d04d5ef3-43e9-48a5-9eb6-aea8aafc1374", "timestamp": "2026-10-29T10:25:24.345Z", "parentId": "4fcb4230-48b8-46b8-af10-3ff5f6b4b0f5"} +{"type": "model.call_start", "data": {"turnId": "0", "model": "claude-sonnet-5.5"}, "ephemeral": true, "id": "5fafe51f-aeab-454d-98b4-9b1b7117eab6", "timestamp": "2026-10-29T10:25:24.353Z", "parentId": "d04d5ef3-43e9-48a5-9eb6-aea8aafc1374"} +{"type": "model.call_finished", "data": {"turnId": "0", "dispatchDurationMs": 1911, "outcome": "success", "editClassifierVersion": 1, "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "containsBuiltInFileEditRequest": false}, "ephemeral": true, "id": "f9bdca72-7ae9-47af-82cf-b7f970e98916", "timestamp": "2026-10-29T10:25:26.273Z", "parentId": "d050a49e-2634-482d-b630-c2f6e0b66e81"} +{"type": "assistant.message", "data": {"messageId": "5a37b8da-3eef-48cf-910d-2a039f6c8478", "originatingMessageId": "4e46f416-a8fd-4fa6-a881-2e5d6ecfd5ce", "model": "claude-sonnet-5.5", "providerId": "copilot", "content": "", "toolRequests": [{"toolCallId": "toolu_017UNqYaNKVUuqruracjbjse", "name": "skill", "arguments": {"skill": "rocm-doctor"}, "type": "function", "intentionSummary": "rocm-doctor"}], "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "turnId": "0", "reasoningText": "The rocm-doctor skill matches this request, so I should invoke it first.", "rte": true, "apiCallId": "msg_011CgWNDr11zvU8ogFi4YpdE", "reasoningBlocks": {"provider": "anthropic-messages", "blocks": [{"signature": "CAQSvwUKEAgTGAI4AUIIdGhpbmtpbmcSDKz3z5jgnnKJsBlT+RoMkmkioUPRhh6J5wcPIjCXUOF8dyABAFrGvkFmO7Pv3MH9fw7sQJtNlkesaqgED24561S2Zt79G7vwxTRYYK8q3ASqlRgP1bMlzEyICUQXkyVAIi7ehjDaXgQyiIq/1rKSeXWcs38f/f3dBsChxtBncheS8e8xIdl/lfm6RHyuU+8KmX9SOfgShdXVLY2BqAsWzJGafFc/qNqn8/A7k2COC6HiYX0PBJZGW3iYIEW8dWx7zX5dR3tzNGS93F3wTSM4y0xuo2f8UqH2Dqb8hqwnjAzYxl73XelZ/2dDrn6ZL7x7lgtsO3tRb+nkT/cPE4H4nH3uKwJJbFrKbAMj/pS7W9TRlYlzhTlEgql0Ec2FmAjL//t0sAy05zglljK+lTDCgSCuZBZikv2MoNwylxwRxaEIyJ2dNfR4Apm0+LSDNzOqZEvEmRBUwGOnERNWkBW18RiAyT/OOhp4+enfO44e3hTBsE23cHomMwAFWKCF3XNiBsMPArHjo9gUcLdv/HEldPa1epwtFPezRflh9Y4VCuYvv9xi/4yx1yJTo3qsiaEiAF/gY3Wsd6rWgEAHdSLOWrfPDpjOhU2Zq0Du+YYlBIeoOVB+g7P+Y6MnwGMKNpaaTy1LElWyT4Um6AMeHlMayArNYCGO3CDDCU0UW3mlK/TZ4Ve71EPQFNIavPUoUa8treu3ZElPiTzzrfhVpkeXD2ruWNZ9rt4yJtZOnjr+kC6OnVLMuEgveqR6I9VXjt7s5FIazzW1eoMgHgfkOW6k3KpsLeOzMpsduPPkFLuE2u2MABNHXtA2CP1nhreQG6wAJ+P5NsKfhGqehcfi+6KmkDt+8WzOXntDHs//cRivjPNJD70dSFYYCnmycd2NtVqUizIDDwmOVS/F8XFfGAE=", "thinking": "The rocm-doctor skill matches this request, so I should invoke it first.\n\n", "type": "thinking"}]}}, "id": "c272b536-5415-4e4a-9d40-eaab4092d891", "timestamp": "2026-10-29T10:25:26.280Z", "parentId": "d050a49e-2634-482d-b630-c2f6e0b66e81"} +{"type": "assistant.reasoning", "data": {"reasoningId": "fa85f623-a034-466c-94de-b38d037b1de8", "content": "The rocm-doctor skill matches this request, so I should invoke it first.", "rte": true}, "ephemeral": true, "id": "42e902cd-90bf-4dc1-9eb4-91d27a3abbf6", "timestamp": "2026-10-29T10:25:26.282Z", "parentId": "c272b536-5415-4e4a-9d40-eaab4092d891"} +{"type": "tool.execution_start", "data": {"toolCallId": "toolu_017UNqYaNKVUuqruracjbjse", "toolName": "skill", "arguments": {"skill": "rocm-doctor"}, "turnId": "0", "model": "claude-sonnet-5.5"}, "id": "40ef35dd-a35a-4b56-8b7a-7fa3f309722e", "timestamp": "2026-10-29T10:25:26.285Z", "parentId": "c272b536-5415-4e4a-9d40-eaab4092d891"} +{"type": "model.call_final_result", "data": {"model": "claude-sonnet-5.5", "isByok": false, "result": "success"}, "ephemeral": true, "id": "e4665bd7-279b-4f50-8505-333a91ee6096", "timestamp": "2026-10-29T10:25:26.286Z", "parentId": "40ef35dd-a35a-4b56-8b7a-7fa3f309722e"} +{"type": "tool.execution_complete", "data": {"toolCallId": "toolu_017UNqYaNKVUuqruracjbjse", "model": "claude-sonnet-5.5", "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "turnId": "0", "rte": true, "success": true, "result": {"content": "Skill \"rocm-doctor\" loaded successfully. Follow the instructions in the skill context.", "detailedContent": "Skill \"rocm-doctor\" loaded successfully. Follow the instructions in the skill context."}, "toolTelemetry": {"properties": {"skillNameHash": "2fbf2aca5c27cfc677c817a2e97d8ab8b806241e2e0d086a052c090397710b8c", "skillSource": "project", "found": "true"}, "restrictedProperties": {"skillName": "rocm-doctor"}, "metrics": {"skillContentLength": 11981, "context_token_limit": 200000, "context_current_tokens": 17620, "large_output_threshold_bytes": 20480, "result_token_count": 22}}}, "id": "cf9a660f-3217-4b21-bc3a-80a4532be2f1", "timestamp": "2026-10-29T10:25:26.292Z", "parentId": "40ef35dd-a35a-4b56-8b7a-7fa3f309722e"} +{"type": "assistant.turn_end", "data": {"turnId": "0"}, "id": "c5ce9fdb-a7b5-4de4-a4cc-ef497dbd3b16", "timestamp": "2026-10-29T10:25:26.300Z", "parentId": "5e21b283-f29a-4689-b40f-ad1f1094f734"} +{"type": "assistant.turn_start", "data": {"turnId": "1", "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e"}, "id": "c73b02a2-c6d9-41bf-8225-d82ec8a3a521", "timestamp": "2026-10-29T10:25:26.302Z", "parentId": "c5ce9fdb-a7b5-4de4-a4cc-ef497dbd3b16"} +{"type": "model.call_start", "data": {"turnId": "1", "model": "claude-sonnet-5.5"}, "ephemeral": true, "id": "f48f62c5-a438-40b6-a94e-3e4bfd45c7c0", "timestamp": "2026-10-29T10:25:26.312Z", "parentId": "c73b02a2-c6d9-41bf-8225-d82ec8a3a521"} +{"type": "assistant.message_start", "data": {"messageId": "31384056-b4a6-44b1-b741-3de7a88e6d90"}, "ephemeral": true, "id": "41280507-2b27-44e8-afdd-c220446bd277", "timestamp": "2026-10-29T10:25:27.698Z", "parentId": "c73b02a2-c6d9-41bf-8225-d82ec8a3a521"} +{"type": "model.call_finished", "data": {"turnId": "1", "dispatchDurationMs": 1705, "outcome": "success", "editClassifierVersion": 1, "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "containsBuiltInFileEditRequest": false}, "ephemeral": true, "id": "bff2e9d6-e1c0-46ad-a77a-8d25a43d73d2", "timestamp": "2026-10-29T10:25:28.027Z", "parentId": "847bc4f8-6bb4-4087-82d6-d22580b2c271"} +{"type": "assistant.message", "data": {"messageId": "31384056-b4a6-44b1-b741-3de7a88e6d90", "originatingMessageId": "4e46f416-a8fd-4fa6-a881-2e5d6ecfd5ce", "model": "claude-sonnet-5.5", "providerId": "copilot", "content": "AMD GPU is stated (hipErrorNoBinaryForGpu is an AMD/HIP error). Checking whether the `rocm` CLI is present.", "toolRequests": [{"toolCallId": "toolu_01174WNLgqGwgY9id6QGafqq", "name": "powershell", "arguments": {"command": "rocm --version", "description": "Check rocm CLI presence"}, "type": "function", "intentionSummary": "Check rocm CLI presence", "toolTitle": "Running command"}], "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "turnId": "1", "rte": true, "apiCallId": "msg_011CgWNDyk6msoXgohUFKz2k"}, "id": "fa2ceada-fdb8-4282-a515-c984a53ddecb", "timestamp": "2026-10-29T10:25:28.030Z", "parentId": "847bc4f8-6bb4-4087-82d6-d22580b2c271"} +{"type": "tool.execution_start", "data": {"toolCallId": "toolu_01174WNLgqGwgY9id6QGafqq", "toolName": "powershell", "arguments": {"command": "rocm --version", "description": "Check rocm CLI presence"}, "turnId": "1", "model": "claude-sonnet-5.5", "toolTitle": "Running command", "shellToolInfo": {"possiblePaths": [], "hasWriteFileRedirection": false}}, "id": "553c4d02-cf48-4b41-9a78-1287704af6f1", "timestamp": "2026-10-29T10:25:28.034Z", "parentId": "fa2ceada-fdb8-4282-a515-c984a53ddecb"} +{"type": "model.call_final_result", "data": {"model": "claude-sonnet-5.5", "isByok": false, "result": "success"}, "ephemeral": true, "id": "a217e152-4162-484e-97cd-479cbfb98509", "timestamp": "2026-10-29T10:25:28.036Z", "parentId": "553c4d02-cf48-4b41-9a78-1287704af6f1"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "cc189d59-c777-4cda-8f91-377211c76428", "timestamp": "2026-10-29T10:25:28.061Z", "parentId": "553c4d02-cf48-4b41-9a78-1287704af6f1"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "2a608139-9d99-4db2-8900-cc0bc12b597c", "timestamp": "2026-10-29T10:25:28.061Z", "parentId": "553c4d02-cf48-4b41-9a78-1287704af6f1"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "62c75ae9-a8ae-42ea-a9c4-a8006c8801c8", "timestamp": "2026-10-29T10:25:28.063Z", "parentId": "553c4d02-cf48-4b41-9a78-1287704af6f1"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "8e720a85-931b-47b4-9cb2-7db2726a3b1b", "timestamp": "2026-10-29T10:25:28.063Z", "parentId": "553c4d02-cf48-4b41-9a78-1287704af6f1"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "74dd3de5-0048-4cf9-bf70-ba6b5a24ec0b", "timestamp": "2026-10-29T10:25:28.064Z", "parentId": "553c4d02-cf48-4b41-9a78-1287704af6f1"} +{"type": "session.background_tasks_changed", "data": {}, "ephemeral": true, "id": "6d30fba6-1c58-483b-ba62-ac9690ac00dd", "timestamp": "2026-10-29T10:25:28.065Z", "parentId": "553c4d02-cf48-4b41-9a78-1287704af6f1"} +{"type": "tool.execution_complete", "data": {"toolCallId": "toolu_01174WNLgqGwgY9id6QGafqq", "model": "claude-sonnet-5.5", "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "turnId": "1", "rte": true, "success": false, "error": {"message": "Permission to run this tool was denied due to the following rules: `shell`", "code": "denied"}, "toolTelemetry": {"properties": {"shell_error_category": "permission_denied"}, "metrics": {"context_token_limit": 200000, "context_current_tokens": 21714, "large_output_threshold_bytes": 20480}}}, "id": "df8f8971-776a-4225-ab2e-6acd8346a10f", "timestamp": "2026-10-29T10:25:28.065Z", "parentId": "553c4d02-cf48-4b41-9a78-1287704af6f1"} +{"type": "assistant.turn_end", "data": {"turnId": "1"}, "id": "39f4c57f-6745-4edd-b282-2e4d3fe02854", "timestamp": "2026-10-29T10:25:28.071Z", "parentId": "df8f8971-776a-4225-ab2e-6acd8346a10f"} +{"type": "assistant.turn_start", "data": {"turnId": "2", "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e"}, "id": "f00439ea-f3d3-442d-97dc-3a94cdff8023", "timestamp": "2026-10-29T10:25:28.072Z", "parentId": "39f4c57f-6745-4edd-b282-2e4d3fe02854"} +{"type": "model.call_start", "data": {"turnId": "2", "model": "claude-sonnet-5.5"}, "ephemeral": true, "id": "fdcf4f1c-b9a5-41b4-8c65-bd924c1143a4", "timestamp": "2026-10-29T10:25:28.081Z", "parentId": "f00439ea-f3d3-442d-97dc-3a94cdff8023"} +{"type": "model.call_finished", "data": {"turnId": "2", "dispatchDurationMs": 2708, "outcome": "success", "editClassifierVersion": 1, "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "containsBuiltInFileEditRequest": false}, "ephemeral": true, "id": "ed4c10a4-153c-4fce-8622-67baf2e3b18f", "timestamp": "2026-10-29T10:25:30.793Z", "parentId": "b0ff25e8-5342-4155-a51b-0bbacd4a50ff"} +{"type": "assistant.message", "data": {"messageId": "9fc5c9a4-347e-4846-af87-0a019dfd45cd", "originatingMessageId": "4e46f416-a8fd-4fa6-a881-2e5d6ecfd5ce", "model": "claude-sonnet-5.5", "providerId": "copilot", "content": "", "toolRequests": [{"toolCallId": "toolu_0119LwSruUym6DVeg6q6x6SB", "name": "grep", "arguments": {"pattern": "NoBinaryForGpu|no kernel image|invalid device function", "paths": "C:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md", "output_mode": "content", "-n": true, "-C": 3}, "type": "function", "intentionSummary": "NoBinaryForGpu|no kernel image|invalid device function", "toolTitle": "Searching files"}], "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "turnId": "2", "reasoningText": "Permission's denied and I shouldn't work around it—instead of improvising, I should check reference.md for information on hipErrorNoBinaryForGpu since the skill advises consulting the reference catalog first.", "rte": true, "apiCallId": "msg_011CgWNE6k37UKBjV2g7tZft", "reasoningBlocks": {"provider": "anthropic-messages", "blocks": [{"signature": "CAQSyQcKEAgTGAI4AUIIdGhpbmtpbmcSDNR+7vBnBrnSMv+MjxoM8HvTHj1Y78IWWpRcIjAr7YNxpvrssADUGUCMEMWDKpL8pWqMm6ZGupHG3v5jgAuXG9VnwB41+YvEZE5fGecq5ga5SO7Ah1zx/sU9Rv40wzgNj9aMwJ338ZR6S7dTjcGTpTTefS148XOGFbcwauYOomj1N0I97/96btsBR7ZrQniT+zVBiaLEOXIHavpyK1+N81GacpGsFqBwxRh/+Hr+XaRImMYkfyDXzJbyrV2iDG4/KpGbBWIQtM7MCn6jsyOYdgmsWBOc/mwXVwmyrvVJf3/YRT+CQQV1RflYruiC2+dmgHnKsAeEOLWmG7NMsbfB7OIJHcP3k/2frvbai2BN9Pck3fRjF/IPz/l3YiFLsj9+A2jHyvimdFd1NI3t36HugRILHgotlN+85V8TX2Ko+xEk95HVvznmMMC8wsz+aYyqvS4+aXYEkCUbqVNTKGp6EaiuaHCR/EJsUc4CErciF4yIqjXX2/JTf0ootW43KpwvpA96L22vhEB0dtFN7oClyylrf5NrlVjvKhJseMFteJ4Xcff5BO6dkSk8AIiM8UXUOPtKTPZM4gFxIvMGy0c2lV+mjkvhp1xjB9oHR1h/1qfA54pRu4AZkrptUsPeURVEe20M9cXBMHuepstCUNk//vwY4QlycmwwJrR8ehHP2pC4bqtxXQjlOQUIcwR+n+3L2wYzB2BGjUJ3iPA5xxmh4cSASphatmH2sp25fr4FZs+yCK6giywU4gJMzD+cRcjq4qTmJnX6wY1F+IhjfdIwzwcyerH4TfOkadgAYNCzGTcpDpnB/1ZA8iPG6QblguHLhMlPXczM7p7Dt9sZO42E7Avl+387F9U/GGNwMP1UFgByO31SZyl7s4zTVU3GfuK+scFSqGpsa8DOKWFvAeZCrkiBF/4vqPk4MIe61tCTzwv/ZsF3fsZRImokcKWhy2y0JhadZJ+dmlxv/CSMmFSG9n0gbWBXCqFKyqzeD6HUJ0KfDaWCZ+dmMV6BaXiJCltctoX8rErKQ58HTvn30i0yaSZ5TM5/zlezxzilVZQPAbgdpN+/K+M8QtCKxYUSXarpMsAzDzaynTLwnTsWMLcsOXqMkyxq74NYRZTWJM07477uP1mP2MJbqD6e2D7G1NJs+hQtGkpKpNVuyJbRVL7LFCYHprjzb/po16ORzcBBoDv6et329iGCGGk2QDGy2F7wmTfZ42EwLQimLK4sZoRr7uG0YQkjSgTh+C4q6TeP9Tmhp3ajC5sYAQ==", "thinking": "Permission's denied and I shouldn't work around it—instead of improvising, I should check reference.md for information on hipErrorNoBinaryForGpu since the skill advises consulting the reference catalog first.\n\n", "type": "thinking"}]}}, "id": "0084ef2f-991f-4fac-a2b3-af6b011fb30d", "timestamp": "2026-10-29T10:25:30.795Z", "parentId": "b0ff25e8-5342-4155-a51b-0bbacd4a50ff"} +{"type": "assistant.reasoning", "data": {"reasoningId": "3164cb53-94f4-4f21-8522-094bb45fb56d", "content": "Permission's denied and I shouldn't work around it—instead of improvising, I should check reference.md for information on hipErrorNoBinaryForGpu since the skill advises consulting the reference catalog first.", "rte": true}, "ephemeral": true, "id": "89fa406f-8479-4b74-9904-699936967275", "timestamp": "2026-10-29T10:25:30.797Z", "parentId": "0084ef2f-991f-4fac-a2b3-af6b011fb30d"} +{"type": "tool.execution_start", "data": {"toolCallId": "toolu_0119LwSruUym6DVeg6q6x6SB", "toolName": "grep", "arguments": {"pattern": "NoBinaryForGpu|no kernel image|invalid device function", "paths": "C:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md", "output_mode": "content", "-n": true, "-C": 3}, "turnId": "2", "model": "claude-sonnet-5.5", "toolTitle": "Searching files"}, "id": "880fbcc5-f655-4566-9637-8ce7426c8ec4", "timestamp": "2026-10-29T10:25:30.798Z", "parentId": "0084ef2f-991f-4fac-a2b3-af6b011fb30d"} +{"type": "model.call_final_result", "data": {"model": "claude-sonnet-5.5", "isByok": false, "result": "success"}, "ephemeral": true, "id": "84b8641f-2058-4268-a9c7-0e641f9d08f7", "timestamp": "2026-10-29T10:25:30.798Z", "parentId": "880fbcc5-f655-4566-9637-8ce7426c8ec4"} +{"type": "tool.execution_complete", "data": {"toolCallId": "toolu_0119LwSruUym6DVeg6q6x6SB", "model": "claude-sonnet-5.5", "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "turnId": "2", "rte": true, "success": true, "result": {"content": "C:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-115-\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-116-| id | OS | Failure mode | Typical signal | Marker (bare-metal Linux) |\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-117-| --- | --- | --- | --- | --- |\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md:118:| `fix-1-arch` | linux/windows/wsl | GPU gfx target not in the framework's build arch list | `hipErrorNoBinaryForGpu`, `HSA_STATUS_ERROR_INVALID_ISA`, \"invalid device function\" | print-only |\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-119-| `fix-2-unset-override` | linux/windows/wsl | `HSA_OVERRIDE_GFX_VERSION` set on a GPU that now has a native wheel | page faults / `OUT_OF_REGISTERS`, override set in env | print-only (auto on windows) |\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-120-| `fix-3-rocm-kernel` | linux | ROCm + distro/kernel form an unsupported triple | ROCm installed but `amdgpu` not loaded; DKMS build failure | print-only |\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-121-| `fix-4-render-group` | linux | User not in `render`/`video` group (or `/dev/kfd` owned by the other group) | cannot open `/dev/kfd`, permission denied | auto |", "detailedContent": "C:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-115-\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-116-| id | OS | Failure mode | Typical signal | Marker (bare-metal Linux) |\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-117-| --- | --- | --- | --- | --- |\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md:118:| `fix-1-arch` | linux/windows/wsl | GPU gfx target not in the framework's build arch list | `hipErrorNoBinaryForGpu`, `HSA_STATUS_ERROR_INVALID_ISA`, \"invalid device function\" | print-only |\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-119-| `fix-2-unset-override` | linux/windows/wsl | `HSA_OVERRIDE_GFX_VERSION` set on a GPU that now has a native wheel | page faults / `OUT_OF_REGISTERS`, override set in env | print-only (auto on windows) |\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-120-| `fix-3-rocm-kernel` | linux | ROCm + distro/kernel form an unsupported triple | ROCm installed but `amdgpu` not loaded; DKMS build failure | print-only |\r\nC:\\Users\\user\\AppData\\Local\\Temp\\probe\\ws\\.claude\\skills\\rocm-doctor\\reference.md-121-| `fix-4-render-group` | linux | User not in `render`/`video` group (or `/dev/kfd` owned by the other group) | cannot open `/dev/kfd`, permission denied | auto |"}, "toolTelemetry": {"properties": {"matches_found": "true", "output_mode": "content", "sandboxApplied": "false", "search_engine": "ripgrep"}, "metrics": {"result_length": 1545, "context_token_limit": 200000, "context_current_tokens": 21898, "large_output_threshold_bytes": 20480, "result_token_count": 590}}}, "id": "ded368be-36c0-402b-95e3-a920074c6e2b", "timestamp": "2026-10-29T10:25:31.068Z", "parentId": "880fbcc5-f655-4566-9637-8ce7426c8ec4"} +{"type": "assistant.turn_end", "data": {"turnId": "2"}, "id": "2b5f4229-5e88-4b47-a55d-1c84100b5fe5", "timestamp": "2026-10-29T10:25:31.071Z", "parentId": "ded368be-36c0-402b-95e3-a920074c6e2b"} +{"type": "assistant.turn_start", "data": {"turnId": "3", "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e"}, "id": "d1ec2cf6-9674-4943-aa75-8eadb7bf645d", "timestamp": "2026-10-29T10:25:31.072Z", "parentId": "2b5f4229-5e88-4b47-a55d-1c84100b5fe5"} +{"type": "model.call_start", "data": {"turnId": "3", "model": "claude-sonnet-5.5"}, "ephemeral": true, "id": "ef46f579-76e9-4381-aa2c-e3ab1aabeb26", "timestamp": "2026-10-29T10:25:31.076Z", "parentId": "d1ec2cf6-9674-4943-aa75-8eadb7bf645d"} +{"type": "assistant.message_start", "data": {"messageId": "da72c13e-a466-4ea8-a334-2794b398dc32"}, "ephemeral": true, "id": "84c239a5-af28-4433-8d3e-4b8303fcddfb", "timestamp": "2026-10-29T10:25:34.724Z", "parentId": "d1ec2cf6-9674-4943-aa75-8eadb7bf645d"} +{"type": "model.call_finished", "data": {"turnId": "3", "dispatchDurationMs": 3773, "outcome": "success", "editClassifierVersion": 1, "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "containsBuiltInFileEditRequest": false}, "ephemeral": true, "id": "ebcc6d00-55d6-48ee-b8c4-ee3cd7a5bde1", "timestamp": "2026-10-29T10:25:34.857Z", "parentId": "2f225d01-e792-4464-a33e-fa6267d4e7ea"} +{"type": "assistant.message", "data": {"messageId": "da72c13e-a466-4ea8-a334-2794b398dc32", "originatingMessageId": "4e46f416-a8fd-4fa6-a881-2e5d6ecfd5ce", "model": "claude-sonnet-5.5", "providerId": "copilot", "content": "I couldn't confirm the cause on your machine. The `rocm` CLI the diagnosis depends on was blocked by a permission rule in this environment, so I ran nothing and changed nothing.\n\n**Most likely cause:** `hipErrorNoBinaryForGpu` matches the catalog entry `fix-1-arch`. Your GPU's `gfx` target isn't in the build list of your PyTorch wheel or ROCm build, so no compiled kernels exist for it. That also explains `torch.cuda.is_available()` returning `False`. I haven't verified this against your hardware.\n\n**Next steps:**\n1. Run `rocm diagnose --symptom \"hipErrorNoBinaryForGpu\" --json` yourself. Check that `has_match` is true and look at the `fix` section.\n2. If `rocm` isn't installed, get it from https://github.com/ROCm/rocm-cli.\n3. For `fix-1-arch`, `rocm fix fix-1-arch` only prints a plan, and you apply it yourself. It won't change anything for you.\n4. Don't set `HSA_OVERRIDE_GFX_VERSION` by hand. It can cause page faults.\n\nIf you'd rather I run the commands, allow shell access and tell me your OS and GPU model.", "toolRequests": [], "interactionId": "19e55712-184c-4d6b-afac-97306eafee2e", "turnId": "3", "rte": true, "apiCallId": "msg_011CgWNEKeBZNcT2bDdoXHfa"}, "id": "497a56a4-a9e8-4ae8-8789-3e78b26bb6bc", "timestamp": "2026-10-29T10:25:34.859Z", "parentId": "2f225d01-e792-4464-a33e-fa6267d4e7ea"} +{"type": "model.call_final_result", "data": {"model": "claude-sonnet-5.5", "isByok": false, "result": "success"}, "ephemeral": true, "id": "7eb92e4a-c9a4-4f8a-94ee-18686fbf0814", "timestamp": "2026-10-29T10:25:34.860Z", "parentId": "497a56a4-a9e8-4ae8-8789-3e78b26bb6bc"} +{"type": "assistant.turn_end", "data": {"turnId": "3"}, "id": "d583e3f7-33d6-46c9-9376-ab5ce0b9e9cc", "timestamp": "2026-10-29T10:25:34.861Z", "parentId": "497a56a4-a9e8-4ae8-8789-3e78b26bb6bc"} +{"type": "session.usage_checkpoint", "data": {"totalNanoAiu": 9283950000, "totalPremiumRequests": 1, "modelCacheState": [{"modelId": "claude-sonnet-5.5", "cacheExpiresAt": "2026-10-29T10:30:31.079Z", "cacheTtlSeconds": 300}], "accountingSnapshot": {"totalPremiumRequestCost": 1, "totalUserRequests": 1, "totalNanoAiu": 9283950000, "aiCreditsStatus": "complete", "tokenDetails": {"input": {"tokenCount": 12}, "cache_read": {"tokenCount": 81780}, "cache_write": {"tokenCount": 30127}, "output": {"tokenCount": 932}}, "totalApiDurationMs": 10091, "sessionStartTime": "2026-10-29T10:25:22.346Z", "codeChanges": {"linesAdded": 0, "linesRemoved": 0, "filesModifiedCount": 0, "filesModified": []}, "modelMetrics": {"claude-sonnet-5.5": {"requests": {"count": 4, "cost": 1}, "usage": {"inputTokens": 111919, "outputTokens": 932, "cacheReadTokens": 81780, "cacheWriteTokens": 30127, "reasoningTokens": 102}, "cacheExpiresAt": "2026-10-29T10:30:31.079Z", "totalNanoAiu": 9283950000, "aiCreditsStatus": "complete", "tokenDetails": {"input": {"tokenCount": 12}, "cache_read": {"tokenCount": 81780}, "cache_write": {"tokenCount": 30127}, "output": {"tokenCount": 932}}}}, "agentMetrics": {"main": {"totalApiDurationMs": 10091, "totalNanoAiu": 9283950000, "modelMetrics": {"claude-sonnet-5.5": {"requests": {"count": 4, "cost": 1}, "usage": {"inputTokens": 111919, "outputTokens": 932, "cacheReadTokens": 81780, "cacheWriteTokens": 30127, "reasoningTokens": 102}, "totalNanoAiu": 9283950000, "tokenDetails": {"input": {"tokenCount": 12}, "cache_read": {"tokenCount": 81780}, "cache_write": {"tokenCount": 30127}, "output": {"tokenCount": 932}}}}}}, "currentModel": "claude-sonnet-5.5", "lastCallInputTokens": 30129, "lastCallOutputTokens": 425}, "usageAccountingWatermarks": {"f4c57a8e-5ec5-4e5d-8d78-f5c383f3ff6a": 4}}, "id": "ddcf5276-0ba8-4fa9-986b-9aa0e512f122", "timestamp": "2026-10-29T10:25:34.866Z", "parentId": "d583e3f7-33d6-46c9-9376-ab5ce0b9e9cc"} +{"type": "assistant.idle", "data": {}, "ephemeral": true, "id": "bb19f403-b4b2-4c74-85aa-8de2a20ab2d7", "timestamp": "2026-10-29T10:25:34.867Z", "parentId": "ddcf5276-0ba8-4fa9-986b-9aa0e512f122"} +{"type": "result", "timestamp": "2026-10-29T10:25:34.879Z", "sessionId": "f4c57a8e-5ec5-4e5d-8d78-f5c383f3ff6a", "exitCode": 0, "usage": {"premiumRequests": 1, "totalApiDurationMs": 10091, "sessionDurationMs": 12533, "codeChanges": {"linesAdded": 0, "linesRemoved": 0, "filesModified": []}}} diff --git a/tests/test_copilot_harness.py b/tests/test_copilot_harness.py new file mode 100644 index 0000000..8eee25d --- /dev/null +++ b/tests/test_copilot_harness.py @@ -0,0 +1,294 @@ +# Copyright Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: MIT + +"""GitHub Copilot CLI as a harness: what is particular to it. + +The shared contract (`tests/test_harness.py`) already runs against the +recordings in `tests/fixtures/harness/copilot/`. These tests cover what only +this harness does: the command it is started with, the closing answer it has to +remember, how its events become tool calls, and where its token counts come +from. The recordings are real output from CLI 1.0.105; the synthetic events +below exist only for shapes the recordings do not contain, and say so. +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from skillscope import harness, routing, usage +from skillscope.engine import no_sandbox + +FIXTURES = Path(__file__).parent / "fixtures" / "harness" / "copilot" + + +def recorded(name: str) -> list[dict]: + lines = (FIXTURES / f"{name}.jsonl").read_text(encoding="utf-8").splitlines() + return [json.loads(line) for line in lines if line.strip()] + + +def translate(events: list[dict]) -> list[dict]: + normalize = harness.get("copilot").normalizer() + return [canonical for event in events for canonical in normalize(event)] + + +def tool_calls(events: list[dict]) -> list[tuple[str, dict]]: + return [ + (name, json.loads(arguments)) + for event in events + for name, arguments in routing._iter_tool_uses(event) + ] + + +class TestTheCommand(unittest.TestCase): + def command(self, **overrides) -> list[str]: + spec = harness.RunSpec(workspace="/ws", prompt="do it", **overrides) + return harness.get("copilot").command(spec) + + def test_the_command_is_headless_json_with_everything_allowed(self) -> None: + self.assertEqual( + self.command(model="claude-sonnet-5.5", effort="high"), + [ + "copilot", + "--output-format", "json", + "--allow-all", + "--no-ask-user", + "--disable-builtin-mcps", + "--model", "claude-sonnet-5.5", + "--reasoning-effort", "high", + ], + ) + + def test_the_prompt_is_not_on_the_command_line(self) -> None: + # It goes in on stdin: no quoting to get wrong on Windows, and no limit + # on its length. + self.assertTrue(harness.get("copilot").prompt_on_stdin) + self.assertNotIn("do it", self.command()) + + def test_model_and_effort_are_left_out_when_not_given(self) -> None: + command = self.command() + self.assertNotIn("--model", command) + self.assertNotIn("--reasoning-effort", command) + + def test_the_callers_flags_come_last(self) -> None: + self.assertEqual(self.command(extra_flags=("--log-level", "none"))[-2:], + ["--log-level", "none"]) + + +class TestTheEnvironment(unittest.TestCase): + def env(self, **overrides) -> dict[str, str]: + return harness.get("copilot").env( + harness.RunSpec(workspace="ws", prompt="p", **overrides) + ) + + def test_a_config_dir_becomes_the_cli_home(self) -> None: + self.assertEqual(self.env(config_dir="cfg")["COPILOT_HOME"], "cfg") + + def test_no_config_dir_leaves_the_runners_own_home_alone(self) -> None: + with mock.patch.dict(os.environ, clear=False): + os.environ.pop("COPILOT_HOME", None) + self.assertNotIn("COPILOT_HOME", self.env()) + + def test_the_cli_does_not_update_itself_mid_run_unless_told_to(self) -> None: + with mock.patch.dict(os.environ, clear=False): + os.environ.pop("COPILOT_AUTO_UPDATE", None) + self.assertEqual(self.env()["COPILOT_AUTO_UPDATE"], "false") + os.environ["COPILOT_AUTO_UPDATE"] = "true" + self.assertEqual(self.env()["COPILOT_AUTO_UPDATE"], "true") + + def test_the_credential_is_the_runners_to_supply_not_the_harnesss_to_carry(self) -> None: + # Not in what the harness sets, so a container is not handed the host's + # token by default; on the host the runner's environment goes underneath. + spec = harness.RunSpec(workspace="ws", prompt="p") + with mock.patch.dict(os.environ, {"GH_TOKEN": "token-for-the-test"}): + self.assertNotIn("GH_TOKEN", self.env()) + self.assertEqual( + harness.get("copilot").host_env(spec)["GH_TOKEN"], "token-for-the-test" + ) + + +class TestWhatTheRecordingBecomes(unittest.TestCase): + def setUp(self) -> None: + self.events = translate(recorded("trigger")) + + def test_the_skill_call_is_the_call_the_engines_read(self) -> None: + [first, *_] = tool_calls(self.events) + self.assertEqual(first, ("skill", {"skill": "rocm-doctor"})) + self.assertEqual( + routing.detect_activation(self.events[0], ["rocm-doctor"]), "rocm-doctor" + ) + + def test_every_call_is_kept_in_order_including_a_denied_one(self) -> None: + # The shell call was denied by a permission rule and is still a call: + # what the agent tried is what an expectation about it is graded on. + self.assertEqual( + [name for name, _ in tool_calls(self.events)], ["skill", "powershell", "grep"] + ) + self.assertEqual(tool_calls(self.events)[1][1]["command"], "rocm --version") + + def test_a_denied_call_reports_why_as_its_result(self) -> None: + results = [ + block["content"] + for event in self.events + if event["type"] == "user" + for block in event["message"]["content"] + ] + self.assertEqual(len(results), 3) + self.assertIn("denied", results[1]) + + def test_streaming_deltas_and_bookkeeping_are_dropped(self) -> None: + kept = {event["type"] for event in self.events} + self.assertLessEqual(kept, {"assistant", "user", "result"}) + + def test_the_run_ends_with_the_answer_it_remembered(self) -> None: + completed, final = no_sandbox.run_completed(self.events) + self.assertTrue(completed) + self.assertEqual(self.events[-1]["subtype"], "success") + # The closing record in the recording carries no answer at all. + closing = recorded("trigger")[-1] + self.assertEqual(closing["type"], "result") + self.assertNotIn("result", closing) + self.assertTrue(final.startswith("I couldn't confirm the cause")) + + def test_the_tokens_reach_the_usage_recorder(self) -> None: + usage.reset() + self.addCleanup(usage.reset) + for event in self.events: + usage.record_stream_event(event) + # From `session.usage_checkpoint`, which reports them once, per model. + self.assertEqual(usage.snapshot().input_tokens, 111919) + self.assertEqual(usage.snapshot().output_tokens, 932) + + def test_a_near_miss_calls_no_skill(self) -> None: + events = translate(recorded("no_trigger")) + self.assertNotIn("skill", [name for name, _ in tool_calls(events)]) + + +class TestShapesTheRecordingsDoNotHave(unittest.TestCase): + """Synthetic: hand-written to the shapes seen elsewhere in the same stream.""" + + def run_events(self, *events: dict) -> list[dict]: + return translate(list(events)) + + def test_a_failed_run_is_reported_as_an_error_result(self) -> None: + [closing] = self.run_events({"type": "result", "exitCode": 1}) + self.assertEqual((closing["subtype"], closing["is_error"]), ("error", True)) + + def test_the_answer_is_the_last_message_with_text(self) -> None: + events = self.run_events( + {"type": "assistant.message", "data": {"content": "first"}}, + {"type": "assistant.message", "data": {"content": "second"}}, + # A message that only requested tools has no text and is not an answer. + {"type": "assistant.message", "data": {"content": ""}}, + {"type": "result", "exitCode": 0}, + ) + self.assertEqual(events[-1]["result"], "second") + + def test_a_sub_agents_message_is_not_the_answer(self) -> None: + events = self.run_events( + {"type": "assistant.message", "data": {"content": "the answer"}}, + {"type": "assistant.message", "agentId": "sub-1", "data": {"content": "side note"}}, + {"type": "result", "exitCode": 0}, + ) + self.assertEqual(events[-1]["result"], "the answer") + + def test_a_stream_that_never_finished_is_not_a_completed_run(self) -> None: + events = self.run_events( + {"type": "tool.execution_start", + "data": {"toolCallId": "c1", "toolName": "skill", "arguments": {"skill": "a"}}}, + ) + self.assertFalse(no_sandbox.run_completed(events)[0]) + + def test_malformed_events_are_ignored_not_fatal(self) -> None: + events = self.run_events( + {"type": "tool.execution_start", "data": "not an object"}, + {"type": "tool.execution_complete"}, + {"type": "session.usage_checkpoint", "data": {"accountingSnapshot": None}}, + {"type": "result", "exitCode": 0}, + ) + self.assertEqual(events[-1]["subtype"], "success") + + def test_tokens_are_summed_across_models(self) -> None: + usage_event = { + "type": "session.usage_checkpoint", + "data": {"accountingSnapshot": {"modelMetrics": { + "model-a": {"usage": {"inputTokens": 10, "outputTokens": 1}}, + "model-b": {"usage": {"inputTokens": 5, "outputTokens": 2}}, + }}}, + } + [spent, _] = self.run_events(usage_event, {"type": "result", "exitCode": 0}) + self.assertEqual(spent["message"]["usage"], {"input_tokens": 15, "output_tokens": 3}) + + def test_a_run_that_reported_no_tokens_adds_no_usage_event(self) -> None: + events = self.run_events({"type": "result", "exitCode": 0}) + self.assertEqual([event["type"] for event in events], ["result"]) + + +class TestTheRecordingThroughTheEnginesStream(unittest.IsolatedAsyncioTestCase): + """The real recording, replayed by a process, read by the engine's own loop.""" + + async def replay(self, stop_when): + # A process that prints the recording, so the subprocess, the line + # reader and the translator all run as they do against the real CLI. + script = ( + "import sys\n" + "for line in open(sys.argv[1], encoding='utf-8'):\n" + " print(line.rstrip('\\n'), flush=True)\n" + ) + recording = str(FIXTURES / "trigger.jsonl") + return await no_sandbox._stream_until( + [sys.executable, "-u", "-c", script, recording], + "", + tempfile.gettempdir(), + dict(os.environ), + stop_when, + harness.get("copilot").normalizer(), + ) + + async def test_the_whole_run_is_read_to_its_end(self) -> None: + events, reason, _, _ = await self.replay(lambda event: None) + self.assertIsNone(reason) + completed, final = no_sandbox.run_completed(events) + self.assertTrue(completed) + self.assertTrue(final) + + async def test_the_routing_decision_stops_the_run_at_the_skill_call(self) -> None: + def stop_when(event: dict) -> str | None: + return "activated" if routing.detect_activation(event, ["rocm-doctor"]) else None + + events, reason, _, _ = await self.replay(stop_when) + self.assertEqual(reason, "activated") + # Stopped on the first canonical event, before the work that followed. + self.assertEqual([name for name, _ in tool_calls(events)], ["skill"]) + + +class TestTheMessagesTheScorersRead(unittest.TestCase): + def setUp(self) -> None: + try: + import inspect_ai # noqa: F401 + except ImportError: + self.skipTest("needs inspect_ai") + + def test_calls_and_their_results_pair_up_with_the_answer_last(self) -> None: + messages, final = no_sandbox.events_to_messages( + translate(recorded("trigger")), "the prompt" + ) + calls = [ + call.function for message in messages + for call in getattr(message, "tool_calls", None) or [] + ] + self.assertEqual(calls, ["skill", "powershell", "grep"]) + results = [m for m in messages if getattr(m, "role", "") == "tool"] + self.assertEqual([m.function for m in results], ["skill", "powershell", "grep"]) + self.assertTrue(final.startswith("I couldn't confirm the cause")) + self.assertEqual(messages[-1].content, final) + + +if __name__ == "__main__": + unittest.main() From 22dcbf3126113ed49007c7cc811bead486665958 Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Thu, 29 Oct 2026 13:39:28 +0200 Subject: [PATCH 5/8] Give a harness the hooks the engines need beyond its events The engines still ask Claude Code directly for things that differ per harness: the model to use when none is named, what a CI run is pinned to, which environment variables a credential can come from, the CLI's spend controls, its version, and whether it can reach its model at all. A second harness could not answer any of them. Add them to the `Harness` contract, with the defaults a harness can usually keep and Claude Code delegating to exactly what the engines did inline: `default_model` and `resolve_model`, `pin_model`, `credentials` and `can_isolate_room`, `cost_flags`, `supports_sandbox`, `version`, and `preflight`, which sends one trivial prompt through the harness's own command so what is checked is what a case will use. Copilot CLI answers them for itself: it defaults to the model that costs one premium request per prompt rather than the largest, which costs fifteen; maps `opus`, `sonnet` and `haiku` onto its own names; pins CI to that one model; authenticates from `COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or `GITHUB_TOKEN`; and has no dollar spend control. The preflight runs the CLI through a runner that kills the whole process tree on timeout. `subprocess.run(timeout=)` kills only the process it launched, and a CLI installed with npm is a shim that starts the real program, so on Windows the program kept the output pipe open and the wait after the kill lasted until it ended on its own: a one second timeout took a minute. Nothing calls the hooks yet. The command line, the preflight, the credential check, the CI pin and the report still take Claude Code as given, and the next change wires them through. All existing tests pass unmodified. New tests run the shared hooks for every registered harness, pin that Claude Code's behaviour is what it was, and drive Copilot's pin, aliases, version and preflight against a stub executable, including a CLI that refuses, one that hangs, and a deadline that has run out. Five deliberate breaks each turn a test red, including reverting to the old timeout handling. Signed-off-by: Eugene Volen --- docs/harnesses.md | 7 + skillscope/harness/base.py | 158 ++++++++++++++++++- skillscope/harness/claude_code.py | 21 +++ skillscope/harness/copilot.py | 30 ++++ tests/test_harness_hooks.py | 252 ++++++++++++++++++++++++++++++ 5 files changed, 467 insertions(+), 1 deletion(-) create mode 100644 tests/test_harness_hooks.py diff --git a/docs/harnesses.md b/docs/harnesses.md index 5f122bb..dd5e713 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -66,6 +66,13 @@ A harness is a subclass of `skillscope.harness.Harness`, registered in | `has_skill_tool` | Whether activating a skill is a tool call in the stream. A harness that activates a skill by reading its `SKILL.md` sets this `False`, and routing reads activation from the file access instead. | | `prompt_on_stdin` | Whether the prompt is written to stdin. A harness that takes it as an argument builds it into `command` and sets this `False`; stdin is closed either way, because some CLIs wait on it. | | `require()` | Refuse to start **on the host** when the harness is not installed there. The default checks `binary` is on the runner's `PATH`. A leg that starts the agent in a container checks the image instead. | +| `default_model`, `resolve_model(model)` | The model used when the caller names none, and how the harness names the one they did. A harness with its own names maps the generic aliases (`opus`, `sonnet`, `haiku`) onto them. | +| `pin_model(model)` | The model a CI run is held to, so results stay comparable run to run. The default pins nothing. | +| `credentials`, `can_isolate_room(environ)` | The environment variables a credential can come from. A routing run keeps the runner's own config and skills out of the session only when the harness authenticates from one of them. | +| `supports_sandbox` | Whether a container leg exists (`--engine sandbox`). | +| `cost_flags(max_budget_usd)` | The CLI's own spend controls for a case. Empty when it has none, which the report then says rather than implying a cap. | +| `version()` | What ` --version` prints, for the report. | +| `preflight(model)` | One trivial prompt through the harness's own command, to prove before any case runs that the credential, the model name and the network work. The default is enough for most harnesses. | | `command(spec)` | The argv that runs one case, headless, streaming JSON events. The executable is named (`binary`), never an absolute path on this machine, so the same command can be started on the host or in a container. | | `env(spec)` | What the harness sets in the process's environment, and nothing else: its own settings, and whatever points it at `spec.config_dir` so the runner's own settings and skills stay out of the session. Not the runner's environment (`PATH`, `HOME`, every key on the machine), which a container must not inherit. Whoever starts the process puts this on top of the environment it has. | | `host_command(spec)`, `host_env(spec)` | What a run started on the host executes: `command(spec)` with its executable found by `shutil.which` (on Windows an npm-installed CLI is a `.cmd` shim that only that finds), and `os.environ` with `env(spec)` on top. A sandbox leg uses `command` and `env` directly. | diff --git a/skillscope/harness/base.py b/skillscope/harness/base.py index b01d6da..bdd55b1 100644 --- a/skillscope/harness/base.py +++ b/skillscope/harness/base.py @@ -23,9 +23,71 @@ import os import shutil +import subprocess +import tempfile from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Callable +from typing import Callable, Mapping + +from .. import deadline + +# What the preflight asks, and how long a CLI gets to answer. A reply of any +# kind proves the credential, the model name and the network; the words are +# only there so the reply is short. +PREFLIGHT_PROMPT = "Reply with the single word: ok" +PREFLIGHT_TIMEOUT_S = 60.0 + +# `--version` output by executable, so a report names the build once however +# many cases ask. +_VERSIONS: dict[str, str | None] = {} + + +def _kill_tree(proc: subprocess.Popen) -> None: + """Kill a process and everything it started. + + `subprocess.run(timeout=)` kills only the process it launched. A CLI + installed with npm is a shim that starts the real program, so killing the + shim leaves the program running with the output pipe open, and the wait that + follows the kill lasts until the program ends on its own. + """ + try: + if os.name == "nt": + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + else: + import signal + + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except (OSError, ProcessLookupError): + pass + try: + proc.kill() + except (OSError, ProcessLookupError): + pass + + +def _run_bounded( + cmd: list[str], *, stdin: str, cwd: str, env: dict[str, str], timeout: float +) -> subprocess.CompletedProcess: + """`subprocess.run` that really stops at `timeout`, children included.""" + spawn: dict = {} + if os.name == "nt": + spawn["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP + else: + spawn["start_new_session"] = True + proc = subprocess.Popen( + cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=cwd, env=env, text=True, encoding="utf-8", **spawn, + ) + try: + out, err = proc.communicate(stdin, timeout=timeout) + except subprocess.TimeoutExpired: + _kill_tree(proc) + proc.communicate() + raise + return subprocess.CompletedProcess(cmd, proc.returncode, out, err) @dataclass(frozen=True) @@ -67,6 +129,100 @@ class Harness(ABC): # it as an argument builds that into `command` and sets this `False`; its # stdin is closed either way, because some CLIs wait on it. prompt_on_stdin: bool = True + # The model used when the caller names none. Not a shared default because + # models are named differently, and cost very differently, per harness. + default_model: str + # The environment variables a credential can come from. A routing run can + # only keep the runner's own configuration out of the session when the + # harness authenticates from one of them; a login held in its own config + # directory would be hidden along with everything else. + credentials: tuple[str, ...] = () + # Whether a container leg exists for this harness (`--engine sandbox`). + supports_sandbox: bool = False + + def resolve_model(self, model: str | None) -> str: + """The model name this harness's CLI takes, for what the caller asked. + + `None` means the caller named none. A harness with its own names maps + the generic aliases (`opus`, `sonnet`, `haiku`) onto them here. + """ + return model or self.default_model + + def pin_model(self, model: str) -> str: + """The model a CI run is held to. + + Automated runs are pinned so results stay comparable from one run to the + next. A harness that pins says so here; the default does not. + """ + return model + + def can_isolate_room(self, environ: Mapping[str, str] | None = None) -> bool: + """Whether the runner's own config can be kept out of a routing session.""" + environ = os.environ if environ is None else environ + return any(environ.get(name, "").strip() for name in self.credentials) + + def cost_flags(self, max_budget_usd: float | None) -> list[str]: + """The CLI's own spend controls for a case, or none when it has none.""" + return [] + + def version(self) -> str | None: + """What ` --version` prints, for the report. `None` if it cannot.""" + executable = shutil.which(self.binary) + if executable is None: + return None + if executable not in _VERSIONS: + try: + proc = subprocess.run( + [executable, "--version"], capture_output=True, text=True, + encoding="utf-8", timeout=30, + ) + lines = [line.strip() for line in (proc.stdout or "").splitlines()] + _VERSIONS[executable] = next((line for line in lines if line), None) + except (OSError, subprocess.SubprocessError): + _VERSIONS[executable] = None + return _VERSIONS[executable] + + def preflight( + self, model: str | None, timeout: float = PREFLIGHT_TIMEOUT_S + ) -> tuple[bool, str]: + """Prove, before any case runs, that the CLI can reach its model. + + One trivial prompt through the harness's own command, so what is checked + is what a case will use: the executable, the credential, the model name + and the network. Returns `(ok, detail)`, with a short reason on failure. + """ + executable = shutil.which(self.binary) + if executable is None: + return False, f"'{self.binary}' CLI not found on PATH" + + bound = deadline.active() + if bound is not None: + leftover = bound.remaining() + if leftover <= 0: + return False, bound.message() + timeout = min(timeout, leftover) + + with tempfile.TemporaryDirectory(prefix="skillscope-preflight-") as workspace, \ + tempfile.TemporaryDirectory(prefix="skillscope-preflight-home-") as home: + spec = RunSpec( + workspace=workspace, prompt=PREFLIGHT_PROMPT, model=model, + config_dir=home, + ) + try: + proc = _run_bounded( + self.host_command(spec), + stdin=PREFLIGHT_PROMPT if self.prompt_on_stdin else "", + cwd=workspace, env=self.host_env(spec), timeout=timeout, + ) + except subprocess.TimeoutExpired: + return False, ( + f"{self.binary} preflight timed out after {timeout:g}s " + "(is the network reachable?)" + ) + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout or f"exit code {proc.returncode}").strip() + return False, detail[:500] + return True, "ok" def require(self) -> None: """Refuse to start on the host when the harness is not installed there. diff --git a/skillscope/harness/claude_code.py b/skillscope/harness/claude_code.py index 9cd89c0..65d38a8 100644 --- a/skillscope/harness/claude_code.py +++ b/skillscope/harness/claude_code.py @@ -12,7 +12,10 @@ from __future__ import annotations +from typing import Mapping + from .. import agent +from .. import routing from .base import Harness, RunSpec @@ -20,6 +23,24 @@ class ClaudeCode(Harness): name = "claude-code" binary = "claude" skills_dir = ".claude/skills" + default_model = "opus" + credentials = routing.ENV_CREDENTIALS + supports_sandbox = True + + def pin_model(self, model: str) -> str: + # CI forces `opus`; the rule and its exemptions live in `agent`. + return agent.enforce_model_policy(model) or model + + def can_isolate_room(self, environ: Mapping[str, str] | None = None) -> bool: + if environ is None: + return routing.can_isolate_config() + return super().can_isolate_room(environ) + + def cost_flags(self, max_budget_usd: float | None) -> list[str]: + # Imported here: the engines import the harness package, not the reverse. + from ..engine import routing as engine_routing + + return engine_routing.host_cost_flags(max_budget_usd) def command(self, spec: RunSpec) -> list[str]: cmd = [ diff --git a/skillscope/harness/copilot.py b/skillscope/harness/copilot.py index 0ebfcf8..a7978a0 100644 --- a/skillscope/harness/copilot.py +++ b/skillscope/harness/copilot.py @@ -39,8 +39,21 @@ import os from typing import Callable +from .. import agent from .base import Harness, RunSpec +# The model a CI run is held to. One premium request per prompt where the largest +# model costs fifteen, which matters for a run that makes dozens of them; and one +# fixed model, so scores stay comparable from one run to the next. +CI_MODEL = "claude-sonnet-5.5" + +# The generic aliases, in this CLI's names. +ALIASES = { + "opus": "claude-opus-5.5", + "sonnet": "claude-sonnet-5.5", + "haiku": "claude-haiku-5.5", +} + def _tool_use(call_id: str, name: str, arguments: dict) -> dict: return { @@ -80,6 +93,23 @@ class Copilot(Harness): name = "copilot" binary = "copilot" skills_dir = ".claude/skills" + default_model = CI_MODEL + credentials = ("COPILOT_GITHUB_TOKEN", "GH_TOKEN", "GITHUB_TOKEN") + + def resolve_model(self, model: str | None) -> str: + wanted = model or self.default_model + return ALIASES.get(wanted.lower(), wanted) + + def pin_model(self, model: str) -> str: + if not agent.is_automated_env() or model == CI_MODEL: + return model + if model.lower().startswith(agent.NO_PROVIDER_PREFIXES): + return model + agent._safe_print( + f"[skillscope] automated run: coercing model '{model}' -> " + f"'{CI_MODEL}' to pin the CI model." + ) + return CI_MODEL def command(self, spec: RunSpec) -> list[str]: # The prompt goes in on stdin, which this CLI reads when no `-p` is diff --git a/tests/test_harness_hooks.py b/tests/test_harness_hooks.py new file mode 100644 index 0000000..598214b --- /dev/null +++ b/tests/test_harness_hooks.py @@ -0,0 +1,252 @@ +# Copyright Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: MIT + +"""What the engines ask of a harness beyond translating its events. + +The model it defaults to and how it names models, the pin a CI run is held to, +what it can authenticate from, its spend controls, its version, and the +preflight that proves it can reach its model before any case runs. The shared +parts run for every registered harness; what is particular to one is tested for +that one, against a stub executable so the real subprocess code runs. +""" + +from __future__ import annotations + +import json +import os +import shutil +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from skillscope import deadline, harness +from skillscope.harness import copilot as copilot_module + + +def stub_cli(test: unittest.TestCase, name: str, script: str) -> None: + """Put an executable called `name` that runs `script` first on PATH. + + A script rather than a mock, so `shutil.which`, the subprocess and its + environment all run. Windows resolves a bare name through PATHEXT, so there + the shim is a `.cmd`; elsewhere it is a `/bin/sh` that execs the interpreter, + which survives an interpreter path with a space in it. + """ + bin_dir = Path(tempfile.mkdtemp(prefix=f"ss-stub-{name}-")) + test.addCleanup(shutil.rmtree, bin_dir, True) + program = bin_dir / f"{name}_stub.py" + program.write_text(script, encoding="utf-8") + if os.name == "nt": + (bin_dir / f"{name}.cmd").write_text( + f'@"{sys.executable}" "{program}" %*\r\n', encoding="utf-8" + ) + else: + shim = bin_dir / name + shim.write_text( + f'#!/bin/sh\nexec "{sys.executable}" "{program}" "$@"\n', encoding="utf-8" + ) + shim.chmod(0o755) + patch = mock.patch.dict( + os.environ, {"PATH": f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '')}"} + ) + patch.start() + test.addCleanup(patch.stop) + + +# A `copilot` that records how it was started, then answers or fails as told. +STUB_COPILOT = """ +import json, os, sys + +if "--version" in sys.argv: + print("GitHub Copilot CLI 9.9.9.") + print("Run 'copilot update' to check for updates.") + sys.exit(0) + +stdin = sys.stdin.read() +record = os.environ.get("STUB_RECORD") +if record: + with open(record, "w", encoding="utf-8") as handle: + json.dump({ + "argv": sys.argv[1:], + "stdin": stdin, + "cwd": os.getcwd(), + "home": os.environ.get("COPILOT_HOME"), + "auto_update": os.environ.get("COPILOT_AUTO_UPDATE"), + }, handle) +if os.environ.get("STUB_FAIL"): + sys.stderr.write(os.environ["STUB_FAIL"]) + sys.exit(1) +if os.environ.get("STUB_HANG"): + import time + time.sleep(60) +print(json.dumps({"type": "result", "exitCode": 0}), flush=True) +""" + + +class TestEveryHarnessHasTheEngineHooks(unittest.TestCase): + def test_a_model_is_named_when_the_caller_names_none(self) -> None: + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + self.assertTrue(subject.default_model) + self.assertEqual(subject.resolve_model(None), subject.default_model) + + def test_a_model_the_harness_does_not_know_passes_through(self) -> None: + for name in harness.names(): + with self.subTest(harness=name): + self.assertEqual(harness.get(name).resolve_model("mockllm/model"), "mockllm/model") + + def test_a_room_can_only_be_isolated_with_a_credential_in_the_environment(self) -> None: + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + self.assertTrue(subject.credentials, "no credential variable is named") + self.assertFalse(subject.can_isolate_room({})) + self.assertFalse(subject.can_isolate_room({subject.credentials[0]: " "})) + for variable in subject.credentials: + self.assertTrue(subject.can_isolate_room({variable: "a-token"}), variable) + + def test_nothing_is_pinned_off_ci(self) -> None: + for name in harness.names(): + with self.subTest(harness=name): + subject = harness.get(name) + with mock.patch.dict(os.environ, clear=False): + for variable in ("CI", "GITHUB_ACTIONS"): + os.environ.pop(variable, None) + self.assertEqual(subject.pin_model("some-model"), "some-model") + + def test_a_no_provider_model_is_never_pinned(self) -> None: + # It reaches no cloud service, so there is no cost or drift to pin. + for name in harness.names(): + with self.subTest(harness=name), mock.patch.dict(os.environ, {"CI": "true"}): + self.assertEqual(harness.get(name).pin_model("mockllm/model"), "mockllm/model") + + +class TestClaudeCodeKeepsItsBehaviour(unittest.TestCase): + """The hooks delegate to what the engine did inline; pin that they still do.""" + + def test_ci_forces_opus_as_it_always_did(self) -> None: + with mock.patch.dict(os.environ, {"CI": "true"}): + self.assertEqual(harness.get("claude-code").pin_model("sonnet"), "opus") + self.assertEqual(harness.get("claude-code").pin_model("opus"), "opus") + + def test_the_default_model_is_the_alias_it_always_was(self) -> None: + self.assertEqual(harness.get("claude-code").resolve_model(None), "opus") + + def test_the_credential_check_follows_the_environment(self) -> None: + with mock.patch.dict(os.environ, clear=False): + for variable in harness.get("claude-code").credentials: + os.environ.pop(variable, None) + self.assertFalse(harness.get("claude-code").can_isolate_room()) + os.environ["ANTHROPIC_AUTH_TOKEN"] = "a-token" + self.assertTrue(harness.get("claude-code").can_isolate_room()) + + +class TestCopilotNamesModelsItsOwnWay(unittest.TestCase): + def test_the_generic_aliases_become_its_names(self) -> None: + copilot = harness.get("copilot") + self.assertEqual(copilot.resolve_model("opus"), "claude-opus-5.5") + self.assertEqual(copilot.resolve_model("Sonnet"), "claude-sonnet-5.5") + self.assertEqual(copilot.resolve_model("haiku"), "claude-haiku-5.5") + + def test_an_exact_name_is_left_alone(self) -> None: + self.assertEqual(harness.get("copilot").resolve_model("gpt-6-astra"), "gpt-6-astra") + + def test_the_default_is_the_cheap_model_not_the_largest(self) -> None: + # Measured: one premium request per prompt on the default, fifteen on + # the largest. A caller who names nothing must not get the latter. + self.assertEqual(harness.get("copilot").resolve_model(None), "claude-sonnet-5.5") + + def test_ci_holds_every_run_to_the_one_model(self) -> None: + copilot = harness.get("copilot") + with mock.patch.dict(os.environ, {"CI": "true"}): + self.assertEqual(copilot.pin_model("claude-opus-5.5"), copilot_module.CI_MODEL) + self.assertEqual(copilot.pin_model(copilot_module.CI_MODEL), copilot_module.CI_MODEL) + + def test_a_coercion_is_said_out_loud(self) -> None: + import contextlib + import io + + out = io.StringIO() + with mock.patch.dict(os.environ, {"CI": "true"}), contextlib.redirect_stdout(out): + harness.get("copilot").pin_model("claude-opus-5.5") + self.assertIn("coercing model 'claude-opus-5.5'", out.getvalue()) + + def test_a_dollar_cap_cannot_be_given_to_a_cli_billed_in_requests(self) -> None: + self.assertEqual(harness.get("copilot").cost_flags(5.0), []) + + +class TestTheVersionTheReportNames(unittest.TestCase): + def test_it_is_the_first_line_the_cli_prints(self) -> None: + stub_cli(self, "copilot", STUB_COPILOT) + self.assertEqual(harness.get("copilot").version(), "GitHub Copilot CLI 9.9.9.") + + def test_it_is_none_when_the_cli_is_not_installed(self) -> None: + with mock.patch("shutil.which", lambda _: None): + self.assertIsNone(harness.get("copilot").version()) + + +class TestThePreflight(unittest.TestCase): + def setUp(self) -> None: + stub_cli(self, "copilot", STUB_COPILOT) + self.record = Path(tempfile.mkdtemp(prefix="ss-record-")) / "run.json" + self.addCleanup(shutil.rmtree, self.record.parent, True) + env = mock.patch.dict(os.environ, {"STUB_RECORD": str(self.record)}) + env.start() + self.addCleanup(env.stop) + + def test_a_cli_that_answers_passes(self) -> None: + self.assertEqual(harness.get("copilot").preflight("claude-sonnet-5.5"), (True, "ok")) + + def test_it_uses_the_same_command_a_case_will(self) -> None: + harness.get("copilot").preflight("claude-sonnet-5.5") + ran = json.loads(self.record.read_text(encoding="utf-8")) + self.assertIn("--output-format", ran["argv"]) + self.assertEqual(ran["argv"][ran["argv"].index("--model") + 1], "claude-sonnet-5.5") + self.assertEqual(ran["stdin"], "Reply with the single word: ok") + + def test_it_runs_isolated_from_the_runners_own_config(self) -> None: + harness.get("copilot").preflight("claude-sonnet-5.5") + ran = json.loads(self.record.read_text(encoding="utf-8")) + self.assertTrue(ran["home"], "no throwaway home was given to the CLI") + self.assertEqual(ran["auto_update"], "false") + + def test_a_refusal_says_why(self) -> None: + with mock.patch.dict(os.environ, {"STUB_FAIL": "Error: Bad credentials"}): + ok, detail = harness.get("copilot").preflight("claude-sonnet-5.5") + self.assertFalse(ok) + self.assertIn("Bad credentials", detail) + + def test_a_cli_that_never_answers_times_out(self) -> None: + import time + + started = time.monotonic() + with mock.patch.dict(os.environ, {"STUB_HANG": "1"}): + ok, detail = harness.get("copilot").preflight("claude-sonnet-5.5", timeout=1) + self.assertFalse(ok) + self.assertIn("timed out", detail) + # The stub sleeps for a minute. A CLI installed with npm is a shim that + # starts the real program, and killing only the shim left the program + # holding the output pipe, so the wait after the kill lasted that long. + self.assertLess(time.monotonic() - started, 20) + + def test_a_missing_cli_is_named(self) -> None: + with mock.patch("shutil.which", lambda _: None): + ok, detail = harness.get("copilot").preflight("claude-sonnet-5.5") + self.assertEqual((ok, detail), (False, "'copilot' CLI not found on PATH")) + + def test_it_does_not_outlive_the_commands_own_deadline(self) -> None: + bound = deadline.Deadline(600.0, command="routing") + previous = deadline.use(bound) + self.addCleanup(deadline.use, previous) + # Nothing left of it: a preflight must not start a call it cannot finish. + with mock.patch.object(bound, "remaining", return_value=0.0): + ok, detail = harness.get("copilot").preflight("claude-sonnet-5.5") + self.assertFalse(ok) + self.assertIn("--timeout", detail) + + +if __name__ == "__main__": + unittest.main() From b4a50cb8bddcfc707632c76810098fdfbd4d371a Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Thu, 29 Oct 2026 13:55:07 +0200 Subject: [PATCH 6/8] Choose the agent from the command line, and run routing on Copilot CLI Add `--harness`, which says which agent runs the cases, and keep `--engine` for where it runs. `host` and `sandbox` are the neutral names for the two legs; the older `claude-code-no-sandbox` and `claude-code` keep working and mean Claude Code, so naming a different harness beside one is refused rather than guessed. `$SKILLSCOPE_HARNESS` and `$SKILLSCOPE_ENGINE` supply what the command line does not. Unnamed means `claude-code` on the host, exactly as before. The command line, the preflight, the credential check, the CI model pin, the spend controls and the report now go through the harness instead of assuming Claude Code. `--model` no longer defaults to `opus` in the parser: each harness has its own default and its own names, and the default for Copilot is the model that costs one premium request per prompt, not the one that costs fifteen. Claude Code keeps `opus` and the CI pin it had. A routing run on another harness reaches nothing but that harness's CLI. Its agent is a subprocess and routing has no judge, so inspect is handed a model that reaches no provider instead of an Anthropic one nothing calls. That is what makes a Copilot run need no model credential, only the token the CLI authenticates from; a room with no such token in the environment is refused, as Claude Code's is without its own. Reports gain `harness` and `harness_version`. Claude Code reports its engine as it always did, so earlier reports stay comparable; any other harness reports the neutral names, because the old ones would name an agent that did not run. A dollar cap that a harness cannot enforce is recorded as unable to bind rather than omitted. `behavioral` is refused on a harness other than Claude Code: its expectations are graded by an LLM judge that still goes through an Anthropic model. That, and the room check for a harness whose session does not list its skills, follow. Every existing test passes unmodified. New tests run the real command against a stub `copilot` that replays the recorded session, with no Anthropic credential in the environment, and check the command line it was started with, the model name and the CI pin, the isolated home, the skills staged where it looks, the preflight, and the report. Five deliberate breaks of the wiring each turn a test red. Signed-off-by: Eugene Volen --- docs/harnesses.md | 24 +-- docs/usage.md | 42 +++-- skillscope/cli.py | 159 +++++++++++++++--- skillscope/engine/routing.py | 77 ++++++--- tests/test_harness_cli.py | 315 +++++++++++++++++++++++++++++++++++ 5 files changed, 555 insertions(+), 62 deletions(-) create mode 100644 tests/test_harness_cli.py diff --git a/docs/harnesses.md b/docs/harnesses.md index dd5e713..4c3be4b 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -23,8 +23,7 @@ free to drift apart. Two harnesses are registered, `claude-code` and `copilot`, and the contract exists so that another is an adapter and a set of tests rather than a change to the -engines. Neither can be chosen from the command line yet; see -[What is not yet behind the contract](#what-is-not-yet-behind-the-contract). +engines. `--harness` chooses one; see [usage.md](usage.md#which-engine-grades-a-run). ## The harnesses @@ -130,19 +129,20 @@ from one is not comparable with a score from another. ## What is not yet behind the contract -The contract covers staging, the command, the environment and the events. These -still assume Claude Code, and a second harness will need to move them behind it: +The contract covers staging, the command, the environment, the events, the model, +the credential check, the spend controls and the preflight. These still assume +Claude Code, and a harness other than it cannot yet use them: -* choosing a harness from the command line (there is no `--harness` flag yet); -* the reachability preflight and the credential check that decides whether the - runner's own config can be kept out of a routing run (`ANTHROPIC_*`); -* the spend-limit flags, which are probed from `claude --help`; -* the CI model pin, which forces `opus`; -* the judge, which uses an Anthropic model through `inspect_ai`; +* **the judge** that grades `behavioral` expectations goes through an Anthropic + model whatever agent produced the run, so `behavioral` is refused on any other + harness; +* **the room check**: Claude Code reports at session start which skills it + loaded, and the engine compares that with what was staged. A harness with no + such event leaves the check not run; * **the sandbox leg**: `--engine claude-code` still runs Claude Code through `inspect_swe`, with its own command line, environment, skill staging and event handling. It should start the harness's `command(spec)` in the container, so that each agent is defined once. Until it does, a harness other than Claude Code has no sandboxed leg; -* `.claude/skills` as the path routing treats as a skills inspection, rather than - a harness's own `skills_dir`. +* **`.claude/skills`** as the path routing treats as a skills inspection, rather + than a harness's own `skills_dir`. diff --git a/docs/usage.md b/docs/usage.md index 3ad6a34..84a9e14 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -249,19 +249,39 @@ environment gets one matrix, labels and all. ## Which engine grades a run -`--engine` chooses what actually runs the cases. The dataset, the CLI and the -reports are identical whichever you pick; only the thing driving the agent -changes. +Two choices decide what runs the cases, and they are separate: `--engine` is +*where* the agent runs, and `--harness` is *which* agent. The dataset and the +reports are identical whichever you pick. -| `--engine` | What runs | Runs in | Needs | -| --- | --- | --- | --- | -| `claude-code-no-sandbox` (default) | The real CLI, under `inspect_ai` | the host | the CLI on `PATH` | -| `claude-code` | The real CLI, via `inspect_swe` | a sandbox | `skillscope[verify]`, Linux only | +| `--engine` | Where it runs | Needs | +| --- | --- | --- | +| `host` (default) | the host, under `inspect_ai` | the harness's CLI on `PATH` | +| `sandbox` | a container, via `inspect_swe` | `skillscope[verify]`, Linux only, and a harness with a sandboxed leg | -Both drive the agent a skill is written for, so what differs between them -is **where the agent runs**, not what it is. What it is -- the harness -- is a -separate question, answered in [harnesses.md](harnesses.md). That is the axis worth choosing -along: the host measures the machine as it is, with whatever else is installed +`claude-code-no-sandbox` and `claude-code` are the older spellings of `host` and +`sandbox`. They name an agent as well as a place, so they mean Claude Code, and +combining one with a different `--harness` is refused. + +| `--harness` | Agent | Sandboxed leg | +| --- | --- | --- | +| `claude-code` (default) | Claude Code | yes | +| `copilot` | GitHub Copilot CLI | no | + +`--harness` and `--engine` can also come from `$SKILLSCOPE_HARNESS` and +`$SKILLSCOPE_ENGINE`. Each harness names models its own way and has its own +default; `--model opus`, `sonnet` and `haiku` mean the right thing for every +harness, and CI holds each harness to one model. A report from one harness is its +own result and is not comparable with another's. How a harness works, and how to +add one, is in [harnesses.md](harnesses.md). + +**`copilot` today runs routing only.** It authenticates from +`COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or `GITHUB_TOKEN`, needs no model credential, +and in Actions needs `permissions: copilot-requests: write` in the calling +workflow. `behavioral` is refused on it until the judge that grades its +expectations is available through the harness. Its preflight sends one real +prompt, which is billed like any other. + +Where the agent runs is the axis worth choosing along: the host measures the machine as it is, with whatever else is installed on it, and the sandbox measures the skill alone. When the two disagree, the disagreement is usually a fact about one of those environments rather than about the skill -- which is a thing one engine on its own cannot tell you. diff --git a/skillscope/cli.py b/skillscope/cli.py index 0b9da79..18ac892 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -83,7 +83,8 @@ usage, ) from . import selection as select_module -from .agent import check_api_reachable, enforce_model_policy +from . import harness as harnesses +from .agent import check_api_reachable # The engines a graded run can be driven by. # claude-code -- the real CLI in a sandbox, via inspect_swe (Linux only) @@ -115,6 +116,72 @@ # container runtime, and a default that refuses on Windows is not a default. DEFAULT_ENGINE = "claude-code-no-sandbox" +# The neutral names for where an agent runs. `host` and `sandbox` say only that. +# The older names also say which agent, and still work, meaning Claude Code. +NEUTRAL_ENGINES = {"host": "claude-code-no-sandbox", "sandbox": "claude-code"} + + +def resolve_leg( + engine: str | None, harness_name: str | None +) -> tuple[str, harnesses.Harness]: + """Settle which agent runs, and where, from whatever was said about each. + + The two are separate questions. `--engine` is where (on the host, or in a + container) and `--harness` is what. The older engine names, which carry the + agent in them, keep working and mean Claude Code; naming a different + harness beside one is a contradiction and is refused rather than guessed. + What comes back is the engine in the spelling the rest of the code branches + on, and the harness itself. + """ + explicit = harness_name or os.environ.get("SKILLSCOPE_HARNESS") or None + subject = harnesses.get(explicit) + # Unnamed means `host`, the neutral spelling of DEFAULT_ENGINE: the older + # spelling names Claude Code, which would contradict any other harness. + engine = engine or os.environ.get("SKILLSCOPE_ENGINE") or "host" + + if engine in NEUTRAL_ENGINES: + leg = NEUTRAL_ENGINES[engine] + elif engine in ENGINES: + if explicit and subject.name != harnesses.DEFAULT: + neutral = {v: k for k, v in NEUTRAL_ENGINES.items()}[engine] + raise SystemExit( + f"error: --engine {engine} runs Claude Code, which contradicts " + f"--harness {subject.name}. Use --engine {neutral} to say where " + "the agent runs without saying which one." + ) + subject = harnesses.get(harnesses.DEFAULT) + leg = engine + else: + raise SystemExit( + f"error: unknown engine {engine!r}. Known engines: " + f"{', '.join([*NEUTRAL_ENGINES, *ENGINES])}." + ) + + if leg == "claude-code" and not subject.supports_sandbox: + raise SystemExit( + f"error: the {subject.name} harness has no sandboxed leg. Use " + "--engine host, which runs it on the host." + ) + return leg, subject + + +def _leg_meta(args: argparse.Namespace) -> dict: + """Which agent graded, where it ran, and which build of it, for a report. + + The engine keeps its older spelling for Claude Code, so reports from before + there was a choice stay comparable; any other harness is reported in the + neutral names, because the old ones would name an agent that did not run. + """ + subject = harnesses.get(getattr(args, "harness", None)) + engine = args.engine + if subject.name != harnesses.DEFAULT: + engine = {v: k for k, v in NEUTRAL_ENGINES.items()}.get(engine, engine) + return { + "engine": engine, + "harness": subject.name, + "harness_version": subject.version(), + } + # Where JSON reports land inside the repo under test. One gitignored directory # rather than a path per repo, so a report is always in the same place. RUNS_DIRNAME = Path(".skillscope") / "runs" @@ -378,7 +445,22 @@ def _prepare_graded_run( raise SystemExit(code) selected = _selected_skills(args.skill) _structural_or_exit(selected if scope is None else sorted(set(scope))) - args.model = enforce_model_policy(args.model) or args.model + args.engine, subject = resolve_leg( + getattr(args, "engine", None), getattr(args, "harness", None) + ) + args.harness = subject.name + if subject.name != harnesses.DEFAULT and getattr(args, "command", "") == "behavioral": + # Behavioral cases are graded by an LLM judge, and that judge still goes + # through an Anthropic model whatever agent produced the run. Saying so + # beats running the agent and then failing every judged expectation. + raise SystemExit( + f"error: --harness {subject.name} cannot run behavioral yet: the " + "judge that grades its expectations is not available through this " + "harness. Routing runs on it." + ) + # The model is the harness's to name and to pin: Claude Code keeps `opus` + # and the CI pin it always had. + args.model = subject.pin_model(subject.resolve_model(args.model)) if getattr(args, "engine", DEFAULT_ENGINE) in INSPECT_ENGINES: # The CLI-based reachability probe tests something these engines do not # use, but they still need one of their own: a graded run starts @@ -399,7 +481,13 @@ def _prepare_graded_run( from .engine import pricing as engine_pricing args.pricing_note = engine_pricing.apply() - if not args.skip_preflight: + if not args.skip_preflight and subject.name != harnesses.DEFAULT: + # The harness's own CLI is all a routing run reaches: its agent is a + # subprocess and there is no judge, so no provider is probed. + ok, detail = subject.preflight(args.model) + if not ok: + raise SystemExit(f"error: {subject.binary} not reachable -- {detail}") + elif not args.skip_preflight: from .engine import models as engine_models ok, detail = engine_models.check_reachable(engine_models.resolve(args.model)) @@ -512,7 +600,7 @@ def _finish_routing( list(routing_set), { "model": args.model, - "engine": args.engine, + **_leg_meta(args), "effort": args.effort, "skills": list(routing_set), "extended": args.extended, @@ -546,6 +634,7 @@ def cmd_routing(args: argparse.Namespace) -> int: _empty_room(args) _prepare_graded_run(args, list(routing_set)) + subject = harnesses.get(args.harness) started = time.time() # Pool the routing set's cases: skill Y's positives are skill X's @@ -583,6 +672,7 @@ def cmd_routing(args: argparse.Namespace) -> int: max_budget_usd=args.max_budget_usd, jobs=args.jobs, log_dir=args.keep_logs or None, + harness=subject, ) if args.engine == "claude-code": # A container really was started: `verify.require()` refuses every @@ -653,11 +743,21 @@ def cmd_routing(args: argparse.Namespace) -> int: # the list also carries `--no-session-persistence`, which says nothing # about spend, so `if flags:` would claim an enforced budget for a run # that passed none. - flags = inspect_routing.host_cost_flags(args.max_budget_usd) + flags = subject.cost_flags(args.max_budget_usd) if "--max-budget-usd" in flags: extra["max_budget_usd"] = args.max_budget_usd - extra["max_budget_enforced_by"] = "claude --max-budget-usd" + extra["max_budget_enforced_by"] = f"{subject.binary} --max-budget-usd" extra["max_budget_can_bind"] = True + elif ( + subject.name != harnesses.DEFAULT + and args.max_budget_usd + and args.max_budget_usd > 0 + ): + # Asked for and not available. Saying so is the difference between a + # run that stayed under a cap and a run whose cap could never fire. + extra["max_budget_usd"] = args.max_budget_usd + extra["max_budget_enforced_by"] = None + extra["max_budget_can_bind"] = False extra["optional_cli_flags_used"] = sorted(f for f in flags if f.startswith("--")) return _finish_routing( @@ -674,6 +774,7 @@ def cmd_routing(args: argparse.Namespace) -> int: def cmd_behavioral(args: argparse.Namespace) -> int: skills = _prepare_graded_run(args) + subject = harnesses.get(args.harness) started = time.time() cases = [] @@ -707,14 +808,14 @@ def cmd_behavioral(args: argparse.Namespace) -> int: # through an inspect model provider. from .engine import no_sandbox - no_sandbox.require_local() + no_sandbox.require_local(subject) outcomes = inspect_behavioral.run( skills, gradable, engine_models.resolve(args.model), args.effort, - solver_factory=lambda d: no_sandbox.claude_code_no_sandbox( - args.model, args.effort, d + solver_factory=lambda d: no_sandbox.harness_solver( + subject, args.model, args.effort, d ), ) else: @@ -731,7 +832,7 @@ def cmd_behavioral(args: argparse.Namespace) -> int: outcomes, { "model": args.model, - "engine": args.engine, + **_leg_meta(args), "effort": args.effort, "skills": skills, "extended": args.extended, @@ -828,7 +929,13 @@ def _add_graded_arguments(parser: argparse.ArgumentParser) -> None: ), ) parser.add_argument( - "--model", default="opus", help="Model alias. CI pins this to opus. Default: opus." + "--model", + default=None, + help=( + "Model to run. Each harness has its own default and its own names " + "(`opus`, `sonnet` and `haiku` are understood by all of them). CI " + "pins each harness to one model. Default: opus for claude-code." + ), ) parser.add_argument( "--effort", @@ -854,16 +961,28 @@ def _add_graded_arguments(parser: argparse.ArgumentParser) -> None: ) parser.add_argument( "--engine", - default=os.environ.get("SKILLSCOPE_ENGINE", DEFAULT_ENGINE), - choices=ENGINES, + default=None, + choices=[*NEUTRAL_ENGINES, *ENGINES], + help=( + "Where the agent runs, under inspect_ai. `host` runs it on the " + "host, on any platform. `sandbox` runs it inside a container via " + "inspect_swe, which needs a POSIX guest and so is Linux only, and " + "additionally needs `pip install 'skillscope[verify]'`; only a " + "harness with a sandboxed leg can use it. `claude-code-no-sandbox` " + "and `claude-code` are the older names for those two and mean " + "Claude Code. Default: host, or $SKILLSCOPE_ENGINE." + ), + ) + parser.add_argument( + "--harness", + default=None, + metavar="NAME", help=( - "Which eval engine runs the cases. Both drive the real claude " - "CLI, under inspect_ai; what differs is where it runs. " - "`claude-code-no-sandbox` runs it on the host, on any platform. " - "`claude-code` runs it inside a container via inspect_swe, which " - "needs a POSIX guest and so is Linux only, and additionally " - "needs `pip install 'skillscope[verify]'`. Both run both graded " - "commands. Default: claude-code-no-sandbox, or $SKILLSCOPE_ENGINE." + "The agent that runs the cases: " + + ", ".join(harnesses.names()) + + ". Each is reported as its own result; harnesses send different " + "prompts and offer different tools, so their scores are not " + "comparable. Default: claude-code, or $SKILLSCOPE_HARNESS." ), ) _add_timeout_argument(parser) diff --git a/skillscope/engine/routing.py b/skillscope/engine/routing.py index 8f603f5..374c8d6 100644 --- a/skillscope/engine/routing.py +++ b/skillscope/engine/routing.py @@ -74,8 +74,13 @@ from .. import deadline, routing as routing_core from ..datasets import Case +from .. import harness as harnesses from . import behavioral, convert, models, no_sandbox, sandbox as sandbox_spec, stats +# The inspect model for a run whose agent is a subprocess and whose provider is +# nobody's but the harness's: it reaches no service and needs no credential. +NO_PROVIDER_MODEL = "mockllm/model" + CLAUDE_CODE = "claude-code" NO_SANDBOX = "claude-code-no-sandbox" @@ -111,7 +116,9 @@ ROUTING_MESSAGE_LIMIT = 30 -def require_isolated_room(engine: str) -> None: +def require_isolated_room( + engine: str, harness: harnesses.Harness | None = None +) -> None: """Refuse a host-leg routing run that cannot keep the runner's room clean. The legacy engine warns here and carries on, because it can afford to: it @@ -132,23 +139,37 @@ def require_isolated_room(engine: str) -> None: """ if engine != NO_SANDBOX: return - if routing_core.can_isolate_config(): + subject = harness or harnesses.get() + if subject.can_isolate_room(): return - named = " or ".join(routing_core.ENV_CREDENTIALS) + named = " or ".join(subject.credentials) + who = ( + "--engine claude-code-no-sandbox" + if subject.name == harnesses.DEFAULT + else f"--harness {subject.name}" + ) + # A harness with a container leg has an alternative to offer; one without + # has only the credential. + alternative = ( + ", or use --engine claude-code, whose guest has no user-level skills at all." + if subject.supports_sandbox + else "." + ) raise SystemExit( - "error: --engine claude-code-no-sandbox cannot run a routing leg " + f"error: {who} cannot run a routing leg " f"without {named}. Routing needs the room to hold exactly " "the skills that were asked for, and redirecting the CLI away from " "the runner's own config dir only works when auth comes from the " "environment. Without it every user-level skill on this machine joins " "the room for every case -- and this leg cannot see that happen or " "say so in the report.\n" - f" Set {named}, or use --engine claude-code, whose guest " - "has no user-level skills at all." + f" Set {named}{alternative}" ) -def install_room(skills: dict[str, Path], workspace: str) -> None: +def install_room( + skills: dict[str, Path], workspace: str, skills_dir: str | None = None +) -> None: """Install every skill in the room into one workspace. `no_sandbox.install_skill` installs one, which is all a behavioral case @@ -162,10 +183,10 @@ def install_room(skills: dict[str, Path], workspace: str) -> None: changing it. """ for skill_dir in skills.values(): - no_sandbox.install_skill(skill_dir, workspace) + no_sandbox.install_skill(skill_dir, workspace, skills_dir) -def _install_room_solver(skills: dict[str, Path]): +def _install_room_solver(skills: dict[str, Path], skills_dir: str | None = None): """Solver that stages the whole room before the CLI is started. Chained *ahead* of `no_sandbox.claude_code_no_sandbox`, which stages one @@ -181,7 +202,7 @@ def _install_room_solver(skills: dict[str, Path]): @solver def _install(): async def solve(state, generate): - install_room(skills, await no_sandbox._workspace()) + install_room(skills, await no_sandbox._workspace(), skills_dir) return state return solve @@ -610,6 +631,7 @@ def _solver( max_budget_usd: float | None = None, max_tool_calls: int | None = None, max_inspection_calls: int | None = None, + harness: harnesses.Harness | None = None, ): """The agent that drives one routing case, for the leg that was asked for. @@ -652,12 +674,14 @@ def _solver( # The host leg stages the room itself, because the CLI reads it off the # filesystem it is handed. `model` is the skillscope alias rather than the - # resolved inspect name: this leg is the `claude` CLI's own `--model` flag, - # not a provider lookup. + # resolved inspect name: this leg is the harness's own `--model` flag, not + # a provider lookup. + subject = harness or harnesses.get() return chain( - _install_room_solver(routing_set), - no_sandbox.claude_code_no_sandbox( - model, effort, room[0], config_dir, host_cost_flags(max_budget_usd), + _install_room_solver(routing_set, subject.skills_dir), + no_sandbox.harness_solver( + subject, model, effort, room[0], config_dir, + subject.cost_flags(max_budget_usd), # Names, not the paths `room` holds: the rule matches what a tool # call named against the room's membership, which is how the # approver is given it too. @@ -965,6 +989,7 @@ def build_task( max_tool_calls: int | None = None, max_inspection_calls: int | None = None, max_budget_usd: float | None = None, + harness: harnesses.Harness | None = None, ): """One inspect `Task` for the *room*, with the cases as its samples. @@ -991,6 +1016,7 @@ def build_task( engine, routing_set, model, effort, config_dir, max_budget_usd, budget_for(engine, max_tool_calls), budget_for(engine, max_inspection_calls), + harness, ), # Only the sandboxed leg, because only its tool calls cross inspect's # bridge for an approver to see. The host leg enforces the same rule @@ -1076,6 +1102,7 @@ def run( max_budget_usd: float | None = None, jobs: int | None = None, log_dir: str | None = None, + harness: harnesses.Harness | None = None, ) -> list[routing_core.Outcome]: """Run every routing case against the room. Mirrors `routing.run_case`'s output. @@ -1109,17 +1136,23 @@ def run( "there is nothing to route between." ) + subject = harness or harnesses.get() if engine == CLAUDE_CODE: + if not subject.supports_sandbox: + raise SystemExit( + f"error: the {subject.name} harness has no sandboxed routing leg. " + "This is a bug in skillscope, not in how it was called." + ) from . import verify verify.require() else: - no_sandbox.require_local() + no_sandbox.require_local(subject) # Before the provider check and before a single token: an unisolated room # is not a worse run, it is a different question answered under this one's # name. - require_isolated_room(engine) + require_isolated_room(engine, subject) sandbox_spec.require_provider() @@ -1131,6 +1164,11 @@ def run( ) resolved = models.resolve(model) + if subject.name != harnesses.DEFAULT: + # Inspect's own model is only the judge's, and routing has no judge: the + # agent is the harness's CLI, run as a subprocess. Handing inspect a + # provider here would demand a credential for a model nothing calls. + resolved = NO_PROVIDER_MODEL # A directory per run, not per case: the CLI writes session state here and # the room is the same for every case, so sharing it costs nothing and # saves re-registering the room once per prompt. Removed on the way out -- @@ -1152,6 +1190,7 @@ def run( max_budget_usd, jobs, log_dir, + subject, ) outcomes: list[routing_core.Outcome] = [] @@ -1178,14 +1217,14 @@ def run( def _evaluate( inspect_eval, cases, routing_set, model, effort, engine, resolved, config_dir, case_timeout=None, max_tool_calls=None, max_inspection_calls=None, - max_budget_usd=None, jobs=None, log_dir=None, + max_budget_usd=None, jobs=None, log_dir=None, harness=None, ): """Run the task. Split out so `run` reads as a sequence of decisions.""" return behavioral.run_eval( inspect_eval, build_task( cases, routing_set, model, effort, engine, config_dir, case_timeout, - max_tool_calls, max_inspection_calls, max_budget_usd, + max_tool_calls, max_inspection_calls, max_budget_usd, harness, ), model=resolved, model_args=models.model_args(resolved), diff --git a/tests/test_harness_cli.py b/tests/test_harness_cli.py new file mode 100644 index 0000000..48a54fd --- /dev/null +++ b/tests/test_harness_cli.py @@ -0,0 +1,315 @@ +# Copyright Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: MIT + +"""`--harness` and `--engine`, from the command line down to the subprocess. + +The central test runs a real routing command against a stub `copilot` that +replays the recorded session, with no Anthropic credential anywhere in the +environment. It is the check that a run on another harness really is that +harness end to end: its command line, its model name, its isolated home, the +skills staged where it looks, and a report that names it. +""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import shutil +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from skillscope import cli, harness, usage +from skillscope.engine import sandbox as engine_sandbox +from tests.test_harness_hooks import stub_cli +from tests.test_skillscope import Repo, tier0_dataset + +RECORDING = Path(__file__).parent / "fixtures" / "harness" / "copilot" / "trigger.jsonl" + +# A `copilot` that records how it was started and replays the recorded session. +# Nothing here uses a backslash or a quote that would need escaping twice. +STUB = """ +import json, os, sys + +if "--version" in sys.argv: + print("GitHub Copilot CLI 9.9.9.") + sys.exit(0) + +prompt = sys.stdin.read() +skills = os.path.join(os.getcwd(), ".claude", "skills") +with open(os.environ["STUB_RECORD"], "a", encoding="utf-8") as handle: + handle.write(json.dumps({ + "argv": sys.argv[1:], + "prompt": prompt, + "home": os.environ.get("COPILOT_HOME"), + "staged": sorted(os.listdir(skills)) if os.path.isdir(skills) else None, + "token_present": bool(os.environ.get("GH_TOKEN")), + }) + chr(10)) + +if prompt == "Reply with the single word: ok": + print(json.dumps({"type": "result", "exitCode": 0}), flush=True) + sys.exit(0) + +with open(os.environ["STUB_REPLAY"], encoding="utf-8") as recording: + for line in recording: + print(line.rstrip(chr(10)), flush=True) +""" + +SKILL = "rocm-doctor" # the skill the recording activates + + +class _Copilot(unittest.TestCase): + """A repo with one skill, a stub `copilot` on PATH, and no Anthropic key.""" + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill(SKILL, dataset=tier0_dataset(SKILL)) + self.repo.activate(routing_room=SKILL) + self.repo.chdir() + + stub_cli(self, "copilot", STUB) + scratch = Path(tempfile.mkdtemp(prefix="ss-cli-")) + self.addCleanup(shutil.rmtree, scratch, True) + self.record = scratch / "starts.jsonl" + self.report = scratch / "report.json" + + env = mock.patch.dict( + os.environ, + { + engine_sandbox.SANDBOX_ENV: "local", + "GH_TOKEN": "a-token-for-the-test", + "STUB_RECORD": str(self.record), + "STUB_REPLAY": str(RECORDING), + }, + ) + env.start() + self.addCleanup(env.stop) + for variable in ( + "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "COPILOT_GITHUB_TOKEN", + "GITHUB_TOKEN", "CI", "GITHUB_ACTIONS", "SKILLSCOPE_HARNESS", + "SKILLSCOPE_ENGINE", + ): + os.environ.pop(variable, None) + usage.reset() + self.addCleanup(usage.reset) + + def parse(self, *argv: str): + return cli.build_parser().parse_args(["routing", *argv]) + + def run_routing(self, *argv: str, whole_room: bool = False) -> int: + # One case that should activate the skill and one that should not: enough + # to exercise everything but the totals, at a fraction of the time. + only = () if whole_room else ("--only", f"{SKILL}-yes-0,{SKILL}-no-0") + args = self.parse( + "--output", str(self.report), "--min-accuracy", "0", *only, *argv + ) + with contextlib.redirect_stdout(io.StringIO()): + return cli.cmd_routing(args) + + def starts(self) -> list[dict]: + lines = self.record.read_text(encoding="utf-8").splitlines() + return [json.loads(line) for line in lines if line.strip()] + + def meta(self) -> dict: + return json.loads(self.report.read_text(encoding="utf-8"))["meta"] + + def totals(self) -> dict: + return json.loads(self.report.read_text(encoding="utf-8"))["totals"] + + +class TestARoutingRunOnCopilot(_Copilot): + def test_it_runs_with_no_anthropic_credential(self) -> None: + self.assertEqual(self.run_routing("--harness", "copilot", whole_room=True), 0) + # The five cases of the smallest dataset that clears the coverage bar, + # and the five negatives skillscope adds to every room. The recording + # activates the skill every time, so the three positives are right and + # the seven negatives are false triggers. + self.assertEqual((self.totals()["graded"], self.totals()["passed"]), (10, 3)) + + def test_the_report_names_the_harness_and_where_it_ran(self) -> None: + self.run_routing("--harness", "copilot") + meta = self.meta() + self.assertEqual(meta["harness"], "copilot") + # The older engine names say Claude Code, which did not run. + self.assertEqual(meta["engine"], "host") + self.assertEqual(meta["harness_version"], "GitHub Copilot CLI 9.9.9.") + self.assertTrue(meta["isolated_config_dir"]) + + def test_the_cli_is_started_the_way_the_harness_says(self) -> None: + self.run_routing("--harness", "copilot") + cases = [s for s in self.starts() if s["prompt"] != "Reply with the single word: ok"] + self.assertEqual(len(cases), 2) + for start in cases: + argv = start["argv"] + self.assertEqual(argv[argv.index("--model") + 1], "claude-sonnet-5.5") + self.assertEqual(argv[argv.index("--reasoning-effort") + 1], "high") + self.assertEqual(start["staged"], [SKILL]) + self.assertTrue(start["home"], "no isolated home was given to the CLI") + self.assertTrue(start["token_present"]) + self.assertEqual( + {s["prompt"] for s in cases}, + {f"please do {SKILL} work 0", f"nothing to do with {SKILL} 0"}, + ) + + def test_the_preflight_proves_the_cli_can_answer_first(self) -> None: + self.run_routing("--harness", "copilot") + first = self.starts()[0] + self.assertEqual(first["prompt"], "Reply with the single word: ok") + self.assertEqual(first["argv"][first["argv"].index("--model") + 1], "claude-sonnet-5.5") + + def test_skipping_the_preflight_skips_only_the_preflight(self) -> None: + self.run_routing("--harness", "copilot", "--skip-preflight") + self.assertNotIn("Reply with the single word: ok", [s["prompt"] for s in self.starts()]) + + def test_an_alias_is_understood_in_its_own_names(self) -> None: + self.run_routing("--harness", "copilot", "--model", "haiku", "--skip-preflight") + argv = self.starts()[0]["argv"] + self.assertEqual(argv[argv.index("--model") + 1], "claude-haiku-5.5") + self.assertEqual(self.meta()["model"], "claude-haiku-5.5") + + def test_ci_holds_the_run_to_the_pinned_model(self) -> None: + with mock.patch.dict(os.environ, {"CI": "true"}): + self.run_routing("--harness", "copilot", "--model", "opus", "--skip-preflight") + argv = self.starts()[0]["argv"] + self.assertEqual(argv[argv.index("--model") + 1], "claude-sonnet-5.5") + self.assertEqual(self.meta()["model"], "claude-sonnet-5.5") + + def test_the_harness_can_come_from_the_environment(self) -> None: + with mock.patch.dict(os.environ, {"SKILLSCOPE_HARNESS": "copilot"}): + self.run_routing("--skip-preflight") + self.assertEqual(self.meta()["harness"], "copilot") + + def test_a_dollar_cap_that_cannot_bind_is_reported_as_such(self) -> None: + self.run_routing("--harness", "copilot", "--max-budget-usd", "2", "--skip-preflight") + meta = self.meta() + self.assertEqual(meta["max_budget_usd"], 2.0) + self.assertFalse(meta["max_budget_can_bind"]) + self.assertIsNone(meta["max_budget_enforced_by"]) + + def test_the_default_cap_is_reported_as_unable_to_bind(self) -> None: + # A cap is in force unless one is given, so every run on a harness with + # no dollar control says so rather than implying it stayed under a limit. + self.run_routing("--harness", "copilot", "--skip-preflight") + meta = self.meta() + self.assertEqual(meta["max_budget_usd"], 0.75) + self.assertFalse(meta["max_budget_can_bind"]) + + def test_the_tokens_it_spent_are_counted(self) -> None: + self.run_routing("--harness", "copilot", "--skip-preflight") + # The recording stops at the skill call, before usage is reported, so + # what this proves is that the counters are the harness's own and exist. + self.assertIn("input_tokens", self.meta()) + + +class TestWhatIsRefusedBeforeAnythingRuns(_Copilot): + def refused(self, *argv: str) -> str: + with self.assertRaises(SystemExit) as raised, contextlib.redirect_stdout(io.StringIO()): + self.run_routing(*argv) + self.assertFalse(self.record.exists(), "the CLI was started before the refusal") + return str(raised.exception) + + def test_a_room_that_cannot_be_isolated(self) -> None: + os.environ.pop("GH_TOKEN") + message = self.refused("--harness", "copilot", "--skip-preflight") + self.assertIn("--harness copilot", message) + for variable in harness.get("copilot").credentials: + self.assertIn(variable, message) + # There is no container leg to suggest instead. + self.assertNotIn("--engine claude-code,", message) + + def test_an_unknown_harness(self) -> None: + message = self.refused("--harness", "no-such-agent") + self.assertIn("no-such-agent", message) + self.assertIn("copilot", message) + + def test_an_older_engine_name_that_contradicts_the_harness(self) -> None: + message = self.refused("--harness", "copilot", "--engine", "claude-code-no-sandbox") + self.assertIn("contradicts", message) + self.assertIn("--engine host", message) + + def test_a_harness_with_no_sandboxed_leg_asked_for_one(self) -> None: + message = self.refused("--harness", "copilot", "--engine", "sandbox") + self.assertIn("no sandboxed leg", message) + + def test_behavioral_while_its_judge_is_not_available(self) -> None: + args = cli.build_parser().parse_args(["behavioral", "--harness", "copilot"]) + with self.assertRaises(SystemExit) as raised: + cli._prepare_graded_run(args) + self.assertIn("cannot run behavioral yet", str(raised.exception)) + self.assertFalse(self.record.exists()) + + def test_a_preflight_that_fails_says_why(self) -> None: + with mock.patch.dict(os.environ, {"STUB_REPLAY": str(self.record.parent / "absent")}): + pass # the replay is not reached: the stub answers the preflight itself + with mock.patch.object(harness.get("copilot"), "preflight", + lambda model, timeout=60.0: (False, "Bad credentials")): + message = self.refused("--harness", "copilot") + self.assertIn("copilot not reachable", message) + self.assertIn("Bad credentials", message) + + +class TestResolvingTheLeg(unittest.TestCase): + def setUp(self) -> None: + env = mock.patch.dict(os.environ, clear=False) + env.start() + self.addCleanup(env.stop) + for variable in ("SKILLSCOPE_HARNESS", "SKILLSCOPE_ENGINE"): + os.environ.pop(variable, None) + + def leg(self, engine, harness_name): + leg, subject = cli.resolve_leg(engine, harness_name) + return leg, subject.name + + def test_nothing_said_is_claude_code_on_the_host(self) -> None: + self.assertEqual(self.leg(None, None), ("claude-code-no-sandbox", "claude-code")) + + def test_the_neutral_engine_names(self) -> None: + self.assertEqual(self.leg("host", None), ("claude-code-no-sandbox", "claude-code")) + self.assertEqual(self.leg("sandbox", None), ("claude-code", "claude-code")) + + def test_the_older_engine_names_keep_meaning_claude_code(self) -> None: + self.assertEqual(self.leg("claude-code", None), ("claude-code", "claude-code")) + self.assertEqual( + self.leg("claude-code-no-sandbox", "claude-code"), + ("claude-code-no-sandbox", "claude-code"), + ) + + def test_another_harness_defaults_to_the_host(self) -> None: + self.assertEqual(self.leg(None, "copilot"), ("claude-code-no-sandbox", "copilot")) + self.assertEqual(self.leg("host", "copilot"), ("claude-code-no-sandbox", "copilot")) + + def test_the_environment_supplies_what_the_command_line_did_not(self) -> None: + os.environ["SKILLSCOPE_HARNESS"] = "copilot" + self.assertEqual(self.leg(None, None), ("claude-code-no-sandbox", "copilot")) + os.environ["SKILLSCOPE_ENGINE"] = "claude-code" + # An environment that contradicts itself is refused like a command line. + with self.assertRaises(SystemExit): + cli.resolve_leg(None, None) + + def test_an_unknown_engine_is_refused_by_name(self) -> None: + with self.assertRaises(SystemExit) as raised: + cli.resolve_leg("no-such-engine", None) + self.assertIn("host", str(raised.exception)) + + def test_claude_code_is_reported_in_the_names_it_always_was(self) -> None: + args = cli.build_parser().parse_args(["routing", "--engine", "host"]) + args.engine, args.harness = cli.resolve_leg(args.engine, args.harness)[0], "claude-code" + with mock.patch("shutil.which", lambda _: None): + self.assertEqual( + cli._leg_meta(args), + {"engine": "claude-code-no-sandbox", "harness": "claude-code", + "harness_version": None}, + ) + + def test_the_model_default_is_the_harnesses_own(self) -> None: + for name, expected in (("claude-code", "opus"), ("copilot", "claude-sonnet-5.5")): + with self.subTest(harness=name): + self.assertEqual(harness.get(name).resolve_model(None), expected) + + +if __name__ == "__main__": + unittest.main() From 142b5775b25ddd70b31afa2aa8458fabc3164821 Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Thu, 29 Oct 2026 14:15:55 +0200 Subject: [PATCH 7/8] Give each stub process its own record file in the CLI tests The tests that run a routing command against a stub `copilot` had every stub process append its start record to one shared file. Inspect runs cases concurrently, and appends from two processes to one file are not atomic on Windows, so a record could be overwritten or cut in half: on Windows with Python 3.12 one run read a half-written line and another saw one start where two were expected, while the same tests passed on 3.10 and on Linux. Each process now writes a file of its own, named so they sort in the order the processes started, and the tests read the directory. The tests that assert the CLI was never started now check that the directory is empty. Signed-off-by: Eugene Volen --- tests/test_harness_cli.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/test_harness_cli.py b/tests/test_harness_cli.py index 48a54fd..a35b9b7 100644 --- a/tests/test_harness_cli.py +++ b/tests/test_harness_cli.py @@ -33,7 +33,7 @@ # A `copilot` that records how it was started and replays the recorded session. # Nothing here uses a backslash or a quote that would need escaping twice. STUB = """ -import json, os, sys +import json, os, sys, time if "--version" in sys.argv: print("GitHub Copilot CLI 9.9.9.") @@ -41,14 +41,18 @@ prompt = sys.stdin.read() skills = os.path.join(os.getcwd(), ".claude", "skills") -with open(os.environ["STUB_RECORD"], "a", encoding="utf-8") as handle: +# One file per process, named so they sort in the order they started. Cases run +# concurrently, and appends from two processes to one file are not atomic on +# Windows: a record can be overwritten or cut in half. +name = "%020d-%d.json" % (time.time_ns(), os.getpid()) +with open(os.path.join(os.environ["STUB_RECORD"], name), "w", encoding="utf-8") as handle: handle.write(json.dumps({ "argv": sys.argv[1:], "prompt": prompt, "home": os.environ.get("COPILOT_HOME"), "staged": sorted(os.listdir(skills)) if os.path.isdir(skills) else None, "token_present": bool(os.environ.get("GH_TOKEN")), - }) + chr(10)) + })) if prompt == "Reply with the single word: ok": print(json.dumps({"type": "result", "exitCode": 0}), flush=True) @@ -74,7 +78,8 @@ def setUp(self) -> None: stub_cli(self, "copilot", STUB) scratch = Path(tempfile.mkdtemp(prefix="ss-cli-")) self.addCleanup(shutil.rmtree, scratch, True) - self.record = scratch / "starts.jsonl" + self.record = scratch / "starts" + self.record.mkdir() self.report = scratch / "report.json" env = mock.patch.dict( @@ -111,8 +116,11 @@ def run_routing(self, *argv: str, whole_room: bool = False) -> int: return cli.cmd_routing(args) def starts(self) -> list[dict]: - lines = self.record.read_text(encoding="utf-8").splitlines() - return [json.loads(line) for line in lines if line.strip()] + """How each CLI process was started, in the order they started.""" + return [ + json.loads(path.read_text(encoding="utf-8")) + for path in sorted(self.record.glob("*.json")) + ] def meta(self) -> dict: return json.loads(self.report.read_text(encoding="utf-8"))["meta"] @@ -209,7 +217,7 @@ class TestWhatIsRefusedBeforeAnythingRuns(_Copilot): def refused(self, *argv: str) -> str: with self.assertRaises(SystemExit) as raised, contextlib.redirect_stdout(io.StringIO()): self.run_routing(*argv) - self.assertFalse(self.record.exists(), "the CLI was started before the refusal") + self.assertEqual(self.starts(), [], "the CLI was started before the refusal") return str(raised.exception) def test_a_room_that_cannot_be_isolated(self) -> None: @@ -240,11 +248,9 @@ def test_behavioral_while_its_judge_is_not_available(self) -> None: with self.assertRaises(SystemExit) as raised: cli._prepare_graded_run(args) self.assertIn("cannot run behavioral yet", str(raised.exception)) - self.assertFalse(self.record.exists()) + self.assertEqual(self.starts(), []) def test_a_preflight_that_fails_says_why(self) -> None: - with mock.patch.dict(os.environ, {"STUB_REPLAY": str(self.record.parent / "absent")}): - pass # the replay is not reached: the stub answers the preflight itself with mock.patch.object(harness.get("copilot"), "preflight", lambda model, timeout=60.0: (False, "Bad credentials")): message = self.refused("--harness", "copilot") From 51dcf692184d18a293dbbf689dbb62c0fac292aa Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Thu, 29 Oct 2026 14:35:34 +0200 Subject: [PATCH 8/8] Name the harness in the console lines of a routing run A routing run on another harness still printed `engine=claude-code-no-sandbox` in its summary line and `[claude-code-no-sandbox] routing:` before the cases, naming an agent that did not run. The report was already right. Both console lines now name the harness and the neutral engine; Claude Code's lines are unchanged. The spelling of the engine for a harness lives in one place, shared by the report and the console. A new test checks the progress lines of a Copilot run, and breaking either line makes it fail. Signed-off-by: Eugene Volen --- skillscope/cli.py | 22 +++++++++++++++++----- skillscope/engine/routing.py | 5 ++++- tests/test_harness_cli.py | 22 ++++++++++++++++++++-- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/skillscope/cli.py b/skillscope/cli.py index 18ac892..bc9c406 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -165,6 +165,18 @@ def resolve_leg( return leg, subject +def _engine_label(engine: str, subject: harnesses.Harness) -> str: + """The engine as a report and the console spell it for this harness. + + Claude Code keeps the older name, so reports from before there was a choice + stay comparable. Any other harness gets the neutral name: the old ones name + an agent that did not run. + """ + if subject.name == harnesses.DEFAULT: + return engine + return {v: k for k, v in NEUTRAL_ENGINES.items()}.get(engine, engine) + + def _leg_meta(args: argparse.Namespace) -> dict: """Which agent graded, where it ran, and which build of it, for a report. @@ -173,11 +185,8 @@ def _leg_meta(args: argparse.Namespace) -> dict: neutral names, because the old ones would name an agent that did not run. """ subject = harnesses.get(getattr(args, "harness", None)) - engine = args.engine - if subject.name != harnesses.DEFAULT: - engine = {v: k for k, v in NEUTRAL_ENGINES.items()}.get(engine, engine) return { - "engine": engine, + "engine": _engine_label(args.engine, subject), "harness": subject.name, "harness_version": subject.version(), } @@ -654,7 +663,10 @@ def cmd_routing(args: argparse.Namespace) -> int: # run does not honour described a parallelism nobody chose. `--jobs` still # means what it always did for `structural --external`, which is the one # command that reads it. - print(f"[routing] {len(cases)} cases, model={args.model}, engine={args.engine}") + where = f"engine={_engine_label(args.engine, subject)}" + if subject.name != harnesses.DEFAULT: + where = f"harness={subject.name}, {where}" + print(f"[routing] {len(cases)} cases, model={args.model}, {where}") from .engine import models as engine_models from .engine import routing as inspect_routing diff --git a/skillscope/engine/routing.py b/skillscope/engine/routing.py index 374c8d6..acec458 100644 --- a/skillscope/engine/routing.py +++ b/skillscope/engine/routing.py @@ -1157,8 +1157,11 @@ def run( sandbox_spec.require_provider() skills = list(routing_set) + # Claude Code's lines keep the engine's name; another harness names itself, + # since the engine's name would say an agent ran that did not. + label = engine if subject.name == harnesses.DEFAULT else subject.name print( - f"[{engine}] routing: {len(cases)} case(s), " + f"[{label}] routing: {len(cases)} case(s), " f"{len(skills)} skill(s) installed together", flush=True, ) diff --git a/tests/test_harness_cli.py b/tests/test_harness_cli.py index a35b9b7..9fcac60 100644 --- a/tests/test_harness_cli.py +++ b/tests/test_harness_cli.py @@ -112,8 +112,11 @@ def run_routing(self, *argv: str, whole_room: bool = False) -> int: args = self.parse( "--output", str(self.report), "--min-accuracy", "0", *only, *argv ) - with contextlib.redirect_stdout(io.StringIO()): - return cli.cmd_routing(args) + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = cli.cmd_routing(args) + self.output = out.getvalue() + return code def starts(self) -> list[dict]: """How each CLI process was started, in the order they started.""" @@ -186,6 +189,21 @@ def test_ci_holds_the_run_to_the_pinned_model(self) -> None: self.assertEqual(argv[argv.index("--model") + 1], "claude-sonnet-5.5") self.assertEqual(self.meta()["model"], "claude-sonnet-5.5") + def test_the_console_names_the_harness_that_ran(self) -> None: + # A run on Copilot must not print the name of an engine that means Claude + # Code, in the summary line or in the line that starts the cases. + self.run_routing("--harness", "copilot", "--skip-preflight") + self.assertIn("harness=copilot, engine=host", self.output) + self.assertIn("[copilot] routing:", self.output) + # The progress lines, not the whole output: the closing note about the room + # check still names the engine that can perform it. + progress = [line for line in self.output.splitlines() if line.startswith("[")] + self.assertTrue(progress) + self.assertFalse( + [line for line in progress if "claude-code" in line], + "a progress line names an engine that means Claude Code", + ) + def test_the_harness_can_come_from_the_environment(self) -> None: with mock.patch.dict(os.environ, {"SKILLSCOPE_HARNESS": "copilot"}): self.run_routing("--skip-preflight")