From dbadb2011cc9791bf2d679b5d26652a2444406e5 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:28:18 -0700 Subject: [PATCH 1/2] chore(evals): store eval cases as .ndjson so review tools skip them Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/README.md | 4 ++-- .../{orchestration.jsonl => orchestration.ndjson} | 0 .../{pull-requests.jsonl => pull-requests.ndjson} | 0 .../{visual-preview.jsonl => visual-preview.ndjson} | 0 evals/mcp-tool-use/run.ts | 10 +++++----- 5 files changed, 7 insertions(+), 7 deletions(-) rename evals/mcp-tool-use/cases/{orchestration.jsonl => orchestration.ndjson} (100%) rename evals/mcp-tool-use/cases/{pull-requests.jsonl => pull-requests.ndjson} (100%) rename evals/mcp-tool-use/cases/{visual-preview.jsonl => visual-preview.ndjson} (100%) diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index bdddd8e96485a..8fa2ce7b5f65e 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -14,8 +14,8 @@ pnpm mcp-tool-use --model claude-opus-5-5 --out runs/opus - `ANTHROPIC_API_KEY` in the repo root `.env` pays for Claude models and the Haiku shell simulator. - Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the `vibeproxy` model provider in `~/.codex/config.toml`. Keep the proxy under about 12 samples in flight across all runs. -- Flags: `--case `, `--cases `, `--samples ` (default 2, and a 3rd when the first 2 disagree), `--concurrency ` (samples in flight, default 8), `--max-steps `, `--effort `, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed). -- `--baseline --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run. +- Flags: `--case `, `--cases `, `--samples ` (default 2, and a 3rd when the first 2 disagree), `--concurrency ` (samples in flight, default 8), `--max-steps `, `--effort `, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed). +- `--baseline --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run. - Cost: a full Sonnet run is about $9, and a full Opus run about $17. Codex models through the proxy cost only the shell simulator. ## Add a case diff --git a/evals/mcp-tool-use/cases/orchestration.jsonl b/evals/mcp-tool-use/cases/orchestration.ndjson similarity index 100% rename from evals/mcp-tool-use/cases/orchestration.jsonl rename to evals/mcp-tool-use/cases/orchestration.ndjson diff --git a/evals/mcp-tool-use/cases/pull-requests.jsonl b/evals/mcp-tool-use/cases/pull-requests.ndjson similarity index 100% rename from evals/mcp-tool-use/cases/pull-requests.jsonl rename to evals/mcp-tool-use/cases/pull-requests.ndjson diff --git a/evals/mcp-tool-use/cases/visual-preview.jsonl b/evals/mcp-tool-use/cases/visual-preview.ndjson similarity index 100% rename from evals/mcp-tool-use/cases/visual-preview.jsonl rename to evals/mcp-tool-use/cases/visual-preview.ndjson diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index cca050021dd4a..0175c937c3f43 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -1678,7 +1678,7 @@ const main = Effect.gen(function* () { } const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const readJsonl = (file: string, decode: (line: string) => Effect.Effect) => + const readNdjson = (file: string, decode: (line: string) => Effect.Effect) => fs.readFileString(file).pipe( Effect.map((content) => content.split("\n").filter((line) => line.trim() !== "")), Effect.flatMap((lines) => Effect.forEach(lines, (line) => decode(line))), @@ -1687,10 +1687,10 @@ const main = Effect.gen(function* () { const caseFiles = flags.cases ?? (yield* fs.readDirectory(casesDir)) - .filter((name) => name.endsWith(".jsonl")) + .filter((name) => name.endsWith(".ndjson")) .toSorted() .map((name) => path.join(casesDir, name)); - const cases = (yield* Effect.forEach(caseFiles, (file) => readJsonl(file, decodeCase))).flat(); + const cases = (yield* Effect.forEach(caseFiles, (file) => readNdjson(file, decodeCase))).flat(); const wanted = flags.case?.flatMap((ids) => ids.split(",")); const codex = isCodex(flags.model); const harness = codex ? "codex" : "claude"; @@ -1764,7 +1764,7 @@ const main = Effect.gen(function* () { shellModel: flags["shell-model"], }; const previous = - flags.baseline === undefined ? [] : yield* readJsonl(flags.baseline, decodeBaselineRow); + flags.baseline === undefined ? [] : yield* readNdjson(flags.baseline, decodeBaselineRow); const baseline = new Map(groups(previous, (row) => row.id).map(([id, r]) => [id, passRate(r)])); // Only a case that passed every baseline sample can stop a run; flaky ones are just reported. const passedBefore = (id: string) => baseline.get(id) === 1; @@ -1854,7 +1854,7 @@ const main = Effect.gen(function* () { ); yield* fs.makeDirectory(out, { recursive: true }); const lines = yield* Effect.forEach(rows, (row) => encodeJson(row)); - yield* fs.writeFileString(path.join(out, "results.jsonl"), `${lines.join("\n")}\n`); + yield* fs.writeFileString(path.join(out, "results.ndjson"), `${lines.join("\n")}\n`); yield* fs.writeFileString(path.join(out, "summary.md"), summary); yield* Effect.log(`Wrote ${rows.length} samples to ${out}.\n\n${summary}`); }); From 0a32bc1520172e6fed02c64230b1eea164371403 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:30:00 -0700 Subject: [PATCH 2/2] chore(evals): keep gitignored run results as .jsonl Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/README.md | 2 +- evals/mcp-tool-use/run.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index 8fa2ce7b5f65e..e478b1fec1e5f 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -15,7 +15,7 @@ pnpm mcp-tool-use --model claude-opus-5-5 --out runs/opus - `ANTHROPIC_API_KEY` in the repo root `.env` pays for Claude models and the Haiku shell simulator. - Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the `vibeproxy` model provider in `~/.codex/config.toml`. Keep the proxy under about 12 samples in flight across all runs. - Flags: `--case `, `--cases `, `--samples ` (default 2, and a 3rd when the first 2 disagree), `--concurrency ` (samples in flight, default 8), `--max-steps `, `--effort `, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed). -- `--baseline --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run. +- `--baseline --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run. - Cost: a full Sonnet run is about $9, and a full Opus run about $17. Codex models through the proxy cost only the shell simulator. ## Add a case diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 0175c937c3f43..f65ef239ed7d6 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -1854,7 +1854,7 @@ const main = Effect.gen(function* () { ); yield* fs.makeDirectory(out, { recursive: true }); const lines = yield* Effect.forEach(rows, (row) => encodeJson(row)); - yield* fs.writeFileString(path.join(out, "results.ndjson"), `${lines.join("\n")}\n`); + yield* fs.writeFileString(path.join(out, "results.jsonl"), `${lines.join("\n")}\n`); yield* fs.writeFileString(path.join(out, "summary.md"), summary); yield* Effect.log(`Wrote ${rows.length} samples to ${out}.\n\n${summary}`); });