diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index bdddd8e96485a..e478b1fec1e5f 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -14,7 +14,7 @@ pnpm mcp-tool-use --model claude-opus-5-5 --out runs/opus - `ANTHROPIC_API_KEY` in the repo root `.env` pays for Claude models and the Haiku shell simulator. - Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the `vibeproxy` model provider in `~/.codex/config.toml`. Keep the proxy under about 12 samples in flight across all runs. -- Flags: `--case `, `--cases `, `--samples ` (default 2, and a 3rd when the first 2 disagree), `--concurrency ` (samples in flight, default 8), `--max-steps `, `--effort `, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed). +- Flags: `--case `, `--cases `, `--samples ` (default 2, and a 3rd when the first 2 disagree), `--concurrency ` (samples in flight, default 8), `--max-steps `, `--effort `, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed). - `--baseline --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run. - Cost: a full Sonnet run is about $9, and a full Opus run about $17. Codex models through the proxy cost only the shell simulator. diff --git a/evals/mcp-tool-use/cases/orchestration.jsonl b/evals/mcp-tool-use/cases/orchestration.ndjson similarity index 100% rename from evals/mcp-tool-use/cases/orchestration.jsonl rename to evals/mcp-tool-use/cases/orchestration.ndjson diff --git a/evals/mcp-tool-use/cases/pull-requests.jsonl b/evals/mcp-tool-use/cases/pull-requests.ndjson similarity index 100% rename from evals/mcp-tool-use/cases/pull-requests.jsonl rename to evals/mcp-tool-use/cases/pull-requests.ndjson diff --git a/evals/mcp-tool-use/cases/visual-preview.jsonl b/evals/mcp-tool-use/cases/visual-preview.ndjson similarity index 100% rename from evals/mcp-tool-use/cases/visual-preview.jsonl rename to evals/mcp-tool-use/cases/visual-preview.ndjson diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index cca050021dd4a..f65ef239ed7d6 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -1678,7 +1678,7 @@ const main = Effect.gen(function* () { } const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const readJsonl = (file: string, decode: (line: string) => Effect.Effect) => + const readNdjson = (file: string, decode: (line: string) => Effect.Effect) => fs.readFileString(file).pipe( Effect.map((content) => content.split("\n").filter((line) => line.trim() !== "")), Effect.flatMap((lines) => Effect.forEach(lines, (line) => decode(line))), @@ -1687,10 +1687,10 @@ const main = Effect.gen(function* () { const caseFiles = flags.cases ?? (yield* fs.readDirectory(casesDir)) - .filter((name) => name.endsWith(".jsonl")) + .filter((name) => name.endsWith(".ndjson")) .toSorted() .map((name) => path.join(casesDir, name)); - const cases = (yield* Effect.forEach(caseFiles, (file) => readJsonl(file, decodeCase))).flat(); + const cases = (yield* Effect.forEach(caseFiles, (file) => readNdjson(file, decodeCase))).flat(); const wanted = flags.case?.flatMap((ids) => ids.split(",")); const codex = isCodex(flags.model); const harness = codex ? "codex" : "claude"; @@ -1764,7 +1764,7 @@ const main = Effect.gen(function* () { shellModel: flags["shell-model"], }; const previous = - flags.baseline === undefined ? [] : yield* readJsonl(flags.baseline, decodeBaselineRow); + flags.baseline === undefined ? [] : yield* readNdjson(flags.baseline, decodeBaselineRow); const baseline = new Map(groups(previous, (row) => row.id).map(([id, r]) => [id, passRate(r)])); // Only a case that passed every baseline sample can stop a run; flaky ones are just reported. const passedBefore = (id: string) => baseline.get(id) === 1;