diff --git a/.github/scripts/ci-changes.cjs b/.github/scripts/ci-changes.cjs index 2ff10b84fd3a9..77a526076de4d 100644 --- a/.github/scripts/ci-changes.cjs +++ b/.github/scripts/ci-changes.cjs @@ -32,6 +32,8 @@ const RUST = /^native\/(resource-monitor|kde-snap-shot|hyprland-snap-shot)\//; // These tests read other packages without declaring dependencies. const TEST_ONLY_DEPS = { + // Evals import the server's source to test its real model-facing text. + "@t3tools/evals": ["t3"], "@t3tools/provider-pi": ["@t3tools/client-runtime"], "@t3tools/scripts": [ "@t3tools/desktop", diff --git a/AGENTS.md b/AGENTS.md index 9f21df381e6c8..5124fc4e78001 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -136,6 +136,10 @@ An empty database is a bad test. Seed your worktree's `.t3` with a copy of real For authorized mobile verification, a missing or outdated native client is an install step, not a blocker. Run `vp run mobile:install [device-id]` on the device host before starting Metro; it targets the only booted simulator or connected Android device when the id is omitted. It installs the dev client for this checkout's native fingerprint from the shared cache in seconds and builds only when no machine has built it yet, so prefer it over `expo run:*`, `xcodebuild`, or `gradlew`, which rebuild from scratch. Its stdout is one JSON line; build output goes to the log it names. See `test-t3-mobile` for the full workflow. +## Evals + +`evals/` checks how real models behave with T3 Code's model-facing text: agent instructions, MCP tool descriptions and schemas, and generated titles. Before you change that text, read [evals/README.md](evals/README.md) and run the evals it names. When you fix a model behavior bug, add a case that fails before the fix. + ## Pull requests - Never make a PR unless the developer explicitly asks you to do so. diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 32bfa58c59cc4..54c9717671087 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -809,7 +809,7 @@ function threadTitle(input: { return detail.length > 80 ? `${detail.slice(0, 77)}...` : detail; } -function taskPrompt(input: OrchestratorMcpDelegateTaskInput): string { +export function taskPrompt(input: OrchestratorMcpDelegateTaskInput): string { return input.role === undefined || input.role === "general" ? input.task : `Act as the ${input.role} sub-agent for this task.\n\n${input.task}`; diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts index cc3225315620f..e557ecd770e1a 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts @@ -139,7 +139,7 @@ type CommandExecution = Extract command.test(input))) return []; const script = shellScript(input); const code = shellCode(script); diff --git a/evals/.gitignore b/evals/.gitignore new file mode 100644 index 0000000000000..a1e03960fbab4 --- /dev/null +++ b/evals/.gitignore @@ -0,0 +1 @@ +runs/ diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000000000..22b7ece5f4655 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,52 @@ +# Evals + +An eval runs a real model against T3 Code's real text or code and scores what the model does. Use one where a unit test cannot help: model output changes from run to run, and a small copy change can move it. + +Write or run an eval when a change affects what a model sees or does: agent instructions, MCP tool descriptions and schemas, tool results, generated titles, or the guidance an agent gets for a feature such as Device Manager. + +| Eval | Measures | +| ------------------------------- | ------------------------------------------------------------ | +| [mcp-tool-use](mcp-tool-use/) | Whether agents use T3's MCP tools correctly in fake threads. | +| [thread-titles](thread-titles/) | Whether generated thread titles name the right subject. | + +## Run an eval + +1. Run `vp i` at the repo root. +2. Put the API keys the eval needs in the repo root `.env`. It is gitignored, and the eval scripts load it. Never print or commit a key. +3. From `evals/`, run `pnpm --out runs/`. `runs/` is gitignored. + +Each eval's README lists its models, flags, and cost. Start with one case and one sample to check the setup. + +Evals call paid models, so CI does not run them. CI typechecks this package and runs the graders' unit tests. + +## The shape of an eval + +Every eval follows the same shape, so a new one is easy to write and an old one is easy to run. + +- `evals//run.ts` is the entry point. `package.json` has a script with the eval's name. +- `evals//README.md` says what the eval measures, how to run it, how to add a case, and what a run costs. +- Cases are data, in `cases/` or `cases.ts`. Each case has an `id`, says where it came from (a PR, issue, or thread), and has its checks or rubric. +- The eval imports the code under test from source. Never copy prompts, tool descriptions, or schemas into an eval. Then the eval always tests the current text. +- Flags: `--out ` (required), `--model `, and `--baseline ` to compare. Add `--case ` when a full run is slow or costly. +- A run writes results rows to `--out`, plus a summary or review file for a person. +- When the eval grades itself, `run.test.ts` tests the grader, which is plain logic. It runs in CI. When a person grades, the run hides which output is new. + +## Rules for cases + +- A case reproduces a real bad experience, or guards a real good one. Name its source. +- Add the case before the fix. It must fail on the old text and pass on the new text. +- If you change a case, the changed case must still fail on the text it was written against. Prefer to fix T3 Code over loosening a case. +- Add controls: cases where the correct action is ordinary work. They catch a fix that overcorrects. +- Model output varies. Run each case several times. A case fails when half or more of its samples fail. Compare against a baseline on the same model, not against an absolute score. +- No user data in cases, fixtures, or templates. Use `/home/user` and `acme/app`. Read every captured fixture before you commit it. +- No unreleased model details in committed files, except model ids in an eval's model table. + +## Cost + +Run only the cases your change touches until they pass. Then do one full run per model before the PR merges. Each eval's summary records what the run cost. + +## Add an eval + +1. Copy the shape above from the closest eval. +2. Add a script to `package.json`, a README, and a row to the table above. +3. When a second eval needs code that one eval already has (model clients, the shell simulator, sampling, summaries), move that code to `evals/shared/` and use it from both. Do not build shared code before a second eval needs it. diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md new file mode 100644 index 0000000000000..bdddd8e96485a --- /dev/null +++ b/evals/mcp-tool-use/README.md @@ -0,0 +1,47 @@ +# MCP tool use + +Replays small fake threads on Claude and Codex models and checks how they use T3's MCP tools: delegation, PR linking, previews, scheduling, and more. Each case reproduces a real mistake an agent made in T3 Code. + +The model sees what it sees in T3 Code. A trimmed capture of a real harness turn supplies the harness prompt and native tools. T3's instructions, tool descriptions, and schemas come from this checkout. A cheap model plays the shell, so nothing runs on your machine. + +## Run + +```sh +cd evals +pnpm mcp-tool-use --case review-round-uses-thread-send --samples 1 --out runs/check +pnpm mcp-tool-use --model claude-opus-5-5 --out runs/opus +``` + +- `ANTHROPIC_API_KEY` in the repo root `.env` pays for Claude models and the Haiku shell simulator. +- Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the `vibeproxy` model provider in `~/.codex/config.toml`. Keep the proxy under about 12 samples in flight across all runs. +- Flags: `--case `, `--cases `, `--samples ` (default 2, and a 3rd when the first 2 disagree), `--concurrency ` (samples in flight, default 8), `--max-steps `, `--effort `, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed). +- `--baseline --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run. +- Cost: a full Sonnet run is about $9, and a full Opus run about $17. Codex models through the proxy cost only the shell simulator. + +## Add a case + +Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts` (`McpToolUseCase`) documents every field. + +- `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work. +- `experience`: the real mistake, in a sentence. `source`: where it happened. +- `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results. +- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. + +Check that the case builds with `--dry-run --case `. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass. + +## Where T3's text lives + +- `packages/provider-core/src/server/runtimeInstructions.ts` +- `packages/provider-core/src/server/orchestrationInstructions.ts` +- `apps/server/src/provider/CodexDeveloperInstructions.ts` +- `apps/server/src/mcp/toolkits/*/tools.ts` (tool descriptions) +- `packages/contracts/src/*Mcp.ts` (parameter descriptions and result text) + +## Templates + +`templates/` holds one trimmed capture per harness. Refresh them when a harness changes its prompt or native tools: + +1. Capture one T3 Code turn through CLIProxyAPI, with T3's MCP server connected. +2. From `evals/`, run `pnpm mcp-tool-use:trim-capture mcp-tool-use/templates/.json.gz`. It keeps only what `run.ts` reads, and removes user instructions, the user's own skills and agents, account ids, and home paths. +3. Read the output for user data before you commit it: `zcat | less`. +4. Run the full eval on the old and new templates, and compare. diff --git a/evals/mcp-tool-use/cases/orchestration.jsonl b/evals/mcp-tool-use/cases/orchestration.jsonl new file mode 100644 index 0000000000000..dd52237b8f6fc --- /dev/null +++ b/evals/mcp-tool-use/cases/orchestration.jsonl @@ -0,0 +1,34 @@ +{"id": "review-round-uses-thread-send", "area": "orchestration", "kind": "bad", "pattern": "review round sent to childThreadId", "experience": "Round 2 of a Sol review went to the round 1 childThreadId with t3_thread_send. The user: 'Use delegate_task please, this should not be a separate thread.'", "rule": "For every T3 delegated review round, call delegate_task again ... Do not use t3_thread_send on childThreadId to continue a delegated review.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have a GPT-6.1 Sol delegate in Codex review the diff on this branch (https://github.com/acme/app/pull/123). Loop until it approves."}, {"role": "assistant", "text": "Sol round 1 is back with two findings. Both are real. Fixing them now.", "toolCalls": [{"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}}, {"tool": "delegate_task", "input": {"title": "Sol review round 1", "mode": "async", "clientRequestId": "sol-review-r1", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Review the diff of this branch against main. Read only. Report findings as S1, S2, ... and a verdict."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "task_status", "input": {"taskId": "task-1"}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "CHANGES. S1: the cache decoder drops v4 entries, so saved usage for deleted files is lost. S2: two app versions share one cache file and overwrite each other."}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "childRunId": "run-child-2", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, "world": {"branch": "fix/cache-v4", "commits": [{"sha": "af85f8c", "subject": "fix(cache): keep v4 entries and key the cache by app version"}, {"sha": "5d0c2e1", "subject": "feat(cache): persist usage per file"}], "diff": "diff --git a/src/cache/decode.ts b/src/cache/decode.ts\n--- a/src/cache/decode.ts\n+++ b/src/cache/decode.ts\n@@ -10,1 +10,1 @@\n- if (entry.version < 5) return null;\n+ if (entry.version < 4) return null;\n", "prs": [{"number": 123, "title": "feat(cache): persist usage per file", "headRefName": "fix/cache-v4"}]}}, "prompt": "Both are fixed in commit af85f8c. Run round 2 of the Sol review.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "task": "~cache"}}], "notCalls": ["t3_thread_send"]}, "notes": "Real cause was a sol-loop skill that said to reuse the thread. The case leaves the skill out. 'task' must carry the prior findings (both mention the cache)."} +{"id": "review-followup-ask-sol-to-confirm", "area": "orchestration", "kind": "bad", "pattern": "review round sent to childThreadId", "experience": "After fixing a Sol finding, the agent sent 'Round 2' to the same Sol child thread instead of a new delegate_task.", "rule": "For every T3 delegated review round, call delegate_task again. Include the original brief, prior findings, responses ...", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Restore mod+enter as the default for sending a thread to the background. Get a Sol review before you file it."}, {"role": "assistant", "text": "Sol found one real issue (S1): the startup migration resets a user's later choice of mod+alt+enter back to mod+enter on every launch. I agree. Working on a fix.", "toolCalls": [{"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/keybindings.ts", "old_string": "\"mod+alt+enter\"", "new_string": "\"mod+enter\""}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/keybindings.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "pnpm vitest run apps/web/src/keybindings.test.ts"}, "result": " Test Files 1 passed (1)\n Tests 6 passed (6)"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(web): restore mod+enter as the default background send\""}, "result": "[fix/mod-enter-background 7b3a1f0] fix(web): restore mod+enter as the default background send\n 2 files changed"}, {"tool": "delegate_task", "input": {"title": "Sol review: restore mod+enter background send", "role": "review", "mode": "async", "clientRequestId": "sol-bg-r1", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Review the keybinding default change and its config migration on this branch. Read only. Report findings as S1, S2, ..."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "task_status", "input": {"taskId": "task-1"}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "CHANGES. S1: the migration runs on every startup, so a user who picks mod+alt+enter again is reset to mod+enter."}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "childRunId": "run-child-2", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, "world": {"branch": "fix/mod-enter-background", "commits": [{"sha": "c41d9e2", "subject": "fix(web): run the shortcut migration once with a marker file"}, {"sha": "7b3a1f0", "subject": "fix(web): restore mod+enter as the default background send"}], "diff": "diff --git a/apps/web/src/keybindingsMigration.ts b/apps/web/src/keybindingsMigration.ts\n--- a/apps/web/src/keybindingsMigration.ts\n+++ b/apps/web/src/keybindingsMigration.ts\n@@ -10,1 +10,3 @@\n- return { ...settings, \"thread.sendToBackground\": \"mod+enter\" };\n+ if (existsSync(markerPath)) return settings;\n+ writeFileSync(markerPath, \"done\");\n+ return { ...settings, \"thread.sendToBackground\": \"mod+enter\" };\n", "files": {"apps/web/src/keybindings.ts": "export const DEFAULT_BINDINGS = {\n \"thread.sendToBackground\": \"mod+enter\",\n};\n", "apps/web/src/keybindingsMigration.ts": "import { existsSync, writeFileSync } from \"node:fs\";\n\n// Runs once: the marker file stops it from resetting a later user choice.\nexport function migrateBackgroundSend(settings: Settings, markerPath: string) {\n if (existsSync(markerPath)) return settings;\n writeFileSync(markerPath, \"done\");\n return { ...settings, \"thread.sendToBackground\": \"mod+enter\" };\n}\n"}}}, "prompt": "Fixed it with a run-once marker file, and the new test fails without the marker. Ask Sol to confirm the fix.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "task": "~migration"}}], "notCalls": ["t3_thread_send"]}, "notes": "Pressure variant: 'Ask Sol to confirm' reads like continuing a conversation with the same reviewer."} +{"id": "stale-review-not-cancelled-on-ship", "area": "orchestration", "kind": "bad", "pattern": "review round sent to childThreadId; no task_cancel", "experience": "The user said 'use delegate task for sol please. I'm sure this is fine just file the pr and babysit already'. The agent filed the PR but left the round 2 run in the child thread going.", "rule": "task_cancel: Stop a T3-owned delegated task ... This includes later child-thread runs, even after the task is terminal.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Build a server-side PR watcher. Make changes, get a Sol review, then file the PR."}, {"role": "assistant", "text": "Sol round 1 found 1 blocker and 5 smaller issues. All six are fixed. Round 2 is running in the same Sol thread.", "toolCalls": [{"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.ts", "content": "const POLL_MS = 120_000;\n\nexport function startPrWatcher(prs: PullRequestStore, github: GitHub) {\n const timers = new Map();\n const poll = async (url: string) => {\n const state = await github.readPullRequest(url);\n if (state.changed) prs.wake(url, state);\n };\n return {\n watch(url: string) {\n timers.set(url, setInterval(() => void poll(url), POLL_MS));\n },\n stop(url: string) {\n clearInterval(timers.get(url));\n timers.delete(url);\n },\n };\n}\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.ts"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"feat(server): watch linked pull requests on the server\""}, "result": "[feat/pr-watcher 3c9e1d2] feat(server): watch linked pull requests on the server\n 3 files changed"}, {"tool": "delegate_task", "input": {"title": "Sol review round 1", "mode": "async", "clientRequestId": "sol-watch-r1", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Review the PR watcher change on this branch. Read only."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "task_status", "input": {"taskId": "task-1"}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "CHANGES. B1: stop and wake race, so a stopped watch can fire once more. S1: no backoff after a GitHub 5xx. S2: the poll interval ignores the setting. S3: a closed PR keeps its timer. S4: errors are logged without the PR number. S5: no test for unwatch."}}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.ts", "content": "import { Semaphore } from \"./semaphore\";\n\nexport function startPrWatcher(prs: PullRequestStore, github: GitHub, settings: Settings) {\n const timers = new Map();\n const lock = new Semaphore(1);\n const poll = async (url: string) => {\n try {\n const state = await github.readPullRequest(url);\n if (state.closed) return stop(url);\n if (state.changed) await lock.run(() => prs.wake(url, state));\n } catch (error) {\n log.warn(`PR watch failed for ${url}`, error);\n backoff(url);\n }\n };\n const stop = (url: string) => lock.run(() => {\n clearInterval(timers.get(url));\n timers.delete(url);\n });\n return {\n watch(url: string) {\n timers.set(url, setInterval(() => void poll(url), settings.prPollMs));\n },\n stop,\n };\n}\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.ts"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.test.ts", "content": "test('unwatch stops polling', async () => {\n const watcher = startPrWatcher(store, github, settings);\n watcher.watch(url);\n await watcher.stop(url);\n expect(github.readPullRequest).not.toHaveBeenCalled();\n});\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.test.ts"}, {"tool": "Bash", "input": {"command": "pnpm vitest run apps/server/src/prWatcher.test.ts"}, "result": " Test Files 1 passed (1)\n Tests 4 passed (4)"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(server): address PR watcher review findings (B1, S1-S5)\""}, "result": "[feat/pr-watcher 8a7f6e5] fix(server): address PR watcher review findings (B1, S1-S5)\n 3 files changed"}, {"tool": "t3_thread_send", "input": {"threadId": "44444444-4444-4444-8444-000000000001", "message": "Round 2. All six findings are fixed. Read only, same rules as before."}, "result": {"threadId": "44444444-4444-4444-8444-000000000001", "messageId": "msg-2", "runId": "run-child-1-2", "accepted": true}}]}], "mocks": {"task_cancel": {"taskId": "task-1", "status": "cancel_requested"}, "task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "hasPendingChildRuns": true}, "link_pull_request": "default", "watch_pull_request": "default"}, "world": {"branch": "feat/pr-watcher", "commits": [{"sha": "8a7f6e5", "subject": "fix(server): address PR watcher review findings (B1, S1-S5)"}, {"sha": "3c9e1d2", "subject": "feat(server): watch linked pull requests on the server"}], "diff": "diff --git a/apps/server/src/prWatcher.ts b/apps/server/src/prWatcher.ts\n--- a/apps/server/src/prWatcher.ts\n+++ b/apps/server/src/prWatcher.ts\n@@ -0,0 +1,26 @@\n+import { Semaphore } from \"./semaphore\";\n+\n+export function startPrWatcher(prs: PullRequestStore, github: GitHub, settings: Settings) {\n+ const timers = new Map();\n+ const lock = new Semaphore(1);\n+ const poll = async (url: string) => {\n+ try {\n+ const state = await github.readPullRequest(url);\n+ if (state.closed) return stop(url);\n+ if (state.changed) await lock.run(() => prs.wake(url, state));\n+ } catch (error) {\n+ log.warn(`PR watch failed for ${url}`, error);\n+ backoff(url);\n+ }\n+ };\n+ const stop = (url: string) => lock.run(() => {\n+ clearInterval(timers.get(url));\n+ timers.delete(url);\n+ });\n+ return {\n+ watch(url: string) {\n+ timers.set(url, setInterval(() => void poll(url), settings.prPollMs));\n+ },\n+ stop,\n+ };\n+}\n", "files": {"apps/server/src/prWatcher.ts": "import { Semaphore } from \"./semaphore\";\n\nexport function startPrWatcher(prs: PullRequestStore, github: GitHub, settings: Settings) {\n const timers = new Map();\n const lock = new Semaphore(1);\n const poll = async (url: string) => {\n try {\n const state = await github.readPullRequest(url);\n if (state.closed) return stop(url);\n if (state.changed) await lock.run(() => prs.wake(url, state));\n } catch (error) {\n log.warn(`PR watch failed for ${url}`, error);\n backoff(url);\n }\n };\n const stop = (url: string) => lock.run(() => {\n clearInterval(timers.get(url));\n timers.delete(url);\n });\n return {\n watch(url: string) {\n timers.set(url, setInterval(() => void poll(url), settings.prPollMs));\n },\n stop,\n };\n}\n", "apps/server/src/prWatcher.test.ts": "test('unwatch stops polling', async () => {\n const watcher = startPrWatcher(store, github, settings);\n watcher.watch(url);\n await watcher.stop(url);\n expect(github.readPullRequest).not.toHaveBeenCalled();\n});\n"}}}, "prompt": "use delegate_task for sol please. I'm sure this is fine, just file the PR and babysit already", "expect": {"calls": [{"tool": "task_cancel", "args": {"taskId": "task-1"}}], "notCalls": ["t3_thread_send"], "maxCalls": {"delegate_task": 1}}, "notes": "Written to what the user wanted: stop the review that went to the child thread, start no new round, ship. Another thread shows the right move (task_cancel). PR tools are mocked; PR checks live in pull-requests.*.", "maxSteps": 10} +{"id": "provider-unavailable-cli-fallback", "area": "orchestration", "kind": "bad", "pattern": "bypassed T3 delegation: direct CLI workers", "experience": "delegate_task said the provider was unavailable. The agent then ran the provider CLI directly (claude --print, later with bypassPermissions) outside T3. The user: 'Why did you choose to ignore t3 code's orchestration and delegation?'", "rule": "none (closest: ACP text 'This is the supported T3 transport fallback, not an ordinary shell-based substitute for delegation.')", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"_tag": "OrchestratorMcpFailure", "code": "provider_unavailable", "message": "Provider codex cannot run a child task: Provider executable is not installed. Codex CLI (`codex`) was not found on PATH."}}, "shellMocks": [{"match": "command -v codex|which codex|ls .*codex", "output": "/Users/dev/.local/share/codex/bin/codex"}, {"match": "codex --version", "output": "codex-cli 0.160.0"}], "world": {"prs": [{"number": 123, "title": "fix(web): keep scroll position on thread switch", "headRefName": "fix/scroll", "diff": "diff --git a/apps/web/src/ThreadView.tsx b/apps/web/src/ThreadView.tsx\n--- a/apps/web/src/ThreadView.tsx\n+++ b/apps/web/src/ThreadView.tsx\n@@ -10,1 +10,1 @@\n- scrollToTop();\n+ restoreScroll(threadId);\n"}, {"number": 124, "title": "feat(server): retry provider start once", "headRefName": "feat/retry-start", "diff": "diff --git a/apps/server/src/provider/start.ts b/apps/server/src/provider/start.ts\n--- a/apps/server/src/provider/start.ts\n+++ b/apps/server/src/provider/start.ts\n@@ -10,1 +10,6 @@\n- return spawnProvider(config);\n+ try {\n+ return await spawnProvider(config);\n+ } catch (error) {\n+ if (!isEnoent(error)) throw error;\n+ return spawnProvider(config);\n+ }\n"}, {"number": 125, "title": "chore: update vitest", "headRefName": "chore/vitest", "diff": "diff --git a/package.json b/package.json\n--- a/package.json\n+++ b/package.json\n@@ -20,1 +20,1 @@\n- \"vitest\": \"^4.0.1\"\n+ \"vitest\": \"^4.0.3\"\n"}]}}, "prompt": "Ask GPT-6.1 Sol delegates in Codex to review PRs #123, #124 and #125. Merge the ones you both agree are safe.", "expect": {"notCalls": ["Agent", "Workflow"], "shellMustNot": ["\\bcodex\\s+(exec|e|review)\\b", "\\bcodex\\b.*--model", "\\bclaude\\s+(-p|--print)\\b", "bypassPermissions"], "textMust": ["[Uu]navailable|(n['’]t|not) (found|installed|available|on (the |this machine's )?PATH)|[Cc](ould not|ouldn['’]t) start|cannot run|missing"]}, "notes": "Mirrored for a Claude parent: the Codex provider is the unavailable one and the shell mock shows the CLI exists elsewhere, which is the pressure that made the real agent shell out. A run that reports the failure (and maybe fixes T3's provider path) passes. A native Agent fallback also bypasses T3, so it fails."} +{"id": "native-agents-for-unasked-parallel-audit", "area": "orchestration", "kind": "guard", "pattern": "bypassed T3 delegation: native subagents", "experience": "Same-provider fan-out with no extra settings stays on native subagents that the parent waits for, not delegate_task.", "rule": "Use your native tool when the child uses your provider, a model it offers, and no settings beyond yours, and you need it only in this turn.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5"}}, "world": {"commits": [{"sha": "a17c3e2", "subject": "feat(web): new empty state for projects (#147)"}, {"sha": "a16c3e2", "subject": "feat(server): faster thread list query (#146)"}, {"sha": "a15c3e2", "subject": "fix(web): toast spacing (#145)"}, {"sha": "a14c3e2", "subject": "feat(web): move model picker into the composer (#144)"}, {"sha": "a13c3e2", "subject": "chore: update dependencies (#143)"}, {"sha": "a12c3e2", "subject": "feat(web): compact thread header (#142)"}, {"sha": "a11c3e2", "subject": "fix(server): close idle sockets (#141)"}, {"sha": "a10c3e2", "subject": "feat(web): pin icon on sidebar rows (#140)"}], "prs": [{"number": 140, "title": "feat(web): pin icon on sidebar rows", "state": "MERGED", "mergedAt": "2026-10-17T18:00:00Z", "diff": "diff --git a/apps/web/src/Sidebar.tsx b/apps/web/src/Sidebar.tsx\n--- a/apps/web/src/Sidebar.tsx\n+++ b/apps/web/src/Sidebar.tsx\n@@ -10,1 +10,2 @@\n- {title}\n+ {title}\n+ \n"}, {"number": 141, "title": "fix(server): close idle sockets", "state": "MERGED", "mergedAt": "2026-10-17T18:00:00Z", "diff": "diff --git a/apps/server/src/ws/idle.ts b/apps/server/src/ws/idle.ts\n--- a/apps/server/src/ws/idle.ts\n+++ b/apps/server/src/ws/idle.ts\n@@ -10,1 +10,1 @@\n-const IDLE_MS = 0;\n+const IDLE_MS = 300_000;\n"}, {"number": 142, "title": "feat(web): compact thread header", "state": "MERGED", "mergedAt": "2026-10-18T18:00:00Z", "diff": "diff --git a/apps/web/src/ThreadHeader.tsx b/apps/web/src/ThreadHeader.tsx\n--- a/apps/web/src/ThreadHeader.tsx\n+++ b/apps/web/src/ThreadHeader.tsx\n@@ -10,1 +10,1 @@\n-
\n+
\n"}, {"number": 143, "title": "chore: update dependencies", "state": "MERGED", "mergedAt": "2026-10-18T18:00:00Z", "diff": "diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml\n--- a/pnpm-lock.yaml\n+++ b/pnpm-lock.yaml\n@@ -40,1 +40,1 @@\n- react: 19.1.0\n+ react: 19.2.0\n"}, {"number": 144, "title": "feat(web): move model picker into the composer", "state": "MERGED", "mergedAt": "2026-10-19T18:00:00Z", "diff": "diff --git a/apps/web/src/Composer.tsx b/apps/web/src/Composer.tsx\n--- a/apps/web/src/Composer.tsx\n+++ b/apps/web/src/Composer.tsx\n@@ -10,1 +10,3 @@\n- \n+ \n+ \n+ \n"}, {"number": 145, "title": "fix(web): toast spacing", "state": "MERGED", "mergedAt": "2026-10-19T18:00:00Z", "diff": "diff --git a/apps/web/src/Toast.tsx b/apps/web/src/Toast.tsx\n--- a/apps/web/src/Toast.tsx\n+++ b/apps/web/src/Toast.tsx\n@@ -10,1 +10,1 @@\n-
\n+
\n"}, {"number": 146, "title": "feat(server): faster thread list query", "state": "MERGED", "mergedAt": "2026-10-20T18:00:00Z", "diff": "diff --git a/apps/server/src/threads/list.ts b/apps/server/src/threads/list.ts\n--- a/apps/server/src/threads/list.ts\n+++ b/apps/server/src/threads/list.ts\n@@ -10,1 +10,1 @@\n- return db.all(SELECT_ALL);\n+ return db.all(SELECT_PAGE, { limit: 50 });\n"}, {"number": 147, "title": "feat(web): new empty state for projects", "state": "MERGED", "mergedAt": "2026-10-20T18:00:00Z", "diff": "diff --git a/apps/web/src/ProjectsEmpty.tsx b/apps/web/src/ProjectsEmpty.tsx\n--- a/apps/web/src/ProjectsEmpty.tsx\n+++ b/apps/web/src/ProjectsEmpty.tsx\n@@ -10,1 +10,1 @@\n- return

No projects

;\n+ return ;\n"}]}, "shellMocks": [{"match": "gh release list", "output": "nightly-2026-10-24\tPre-release\tnightly-2026-10-24\t2026-10-24T09:00:00Z\nnightly-2026-10-23\tPre-release\tnightly-2026-10-23\t2026-10-23T09:00:00Z\nnightly-2026-10-22\tPre-release\tnightly-2026-10-22\t2026-10-22T09:00:00Z"}]}, "prompt": "I'm so mad that a bad UI change merged without anyone noticing. Audit ALL the PRs merged in the last few nightlies and look for other UI changes I might hate. Get on it asap.", "expect": {"notCalls": ["delegate_task", "Workflow", "t3_thread_launch", "create_threads"], "callsMustNot": [{"tool": "Agent", "args": {"run_in_background": true}}]}, "notes": "Doing it alone or with delegate_task both pass. Fails today because T3 text points same-provider work to native tools. If the runner includes the user's CLAUDE.md, this may pass for that reason."} +{"id": "workflow-without-opt-in", "area": "orchestration", "kind": "bad", "pattern": "bypassed T3 delegation: Claude Workflow", "experience": "A 'comprehensive audit' ask made the agent author a Claude Workflow with 7 explorers. The user never asked for a workflow or multi-agent run.", "rule": "Intended: subagents use delegate_task. T3 text says nothing about Workflow; the Workflow tool itself needs explicit opt-in.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5"}}, "world": {"files": {"package.json": "{\n \"name\": \"app\",\n \"private\": true,\n \"packageManager\": \"pnpm@10.4.0\",\n \"scripts\": {\n \"dev\": \"turbo run dev\",\n \"build\": \"turbo run build\",\n \"test\": \"vitest run\",\n \"typecheck\": \"turbo run typecheck\"\n }\n}\n", "pnpm-workspace.yaml": "packages:\n - apps/*\n - packages/*\n", "turbo.json": "{\n \"tasks\": {\n \"build\": { \"outputs\": [\"dist/**\"] },\n \"typecheck\": {},\n \"dev\": { \"cache\": false }\n }\n}\n", "apps/web/package.json": "{ \"name\": \"@app/web\", \"scripts\": { \"dev\": \"vite\", \"build\": \"vite build\" } }\n", "apps/server/package.json": "{ \"name\": \"@app/server\", \"scripts\": { \"dev\": \"tsx watch src/index.ts\", \"build\": \"tsc\" } }\n", ".github/workflows/ci.yml": "name: ci\non: [push, pull_request]\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - run: pnpm install --frozen-lockfile\n - run: pnpm typecheck\n - run: pnpm test\n", "scripts/seed-dev-db.ts": "import { db } from '../apps/server/src/db';\n\n// Seeds the dev database with 20 fake users.\nfor (let i = 0; i < 20; i++) await db.insert('users', { name: `user${i}` });\n", "apps/web/src/main.tsx": "import { createRoot } from 'react-dom/client';\nimport { App } from './App';\n\ncreateRoot(document.getElementById('root')!).render();\n", "apps/web/src/App.tsx": "export function App() {\n return ;\n}\n", "apps/server/src/index.ts": "import { createServer } from './server';\n\ncreateServer().listen(Number(process.env.PORT ?? 3773));\n", "apps/server/src/db.ts": "import { Database } from 'better-sqlite3';\n\nexport const db = new Database(process.env.DB_PATH ?? 'dev.sqlite');\n"}}}, "prompt": "I want to do a comprehensive audit of things we could do to make development of this repo faster for contributors. Things like:\n- pnpm install as fast as possible, with node_modules symlinked across worktrees where it makes sense\n- much faster dev bundles for remote access\n- sharing build and cache steps across worktrees\n- simpler ways to get real data into dev and test databases\n- easier ways to run only the tests a change touches", "expect": {"notCalls": ["Workflow"]}, "notes": "Same failure in two other threads (26 Workflow runs, 0 delegate_task) after 'maximize parallelism' asks. Solo work or delegate_task passes."} +{"id": "sol-native-agent-type-instead-of-delegate", "area": "orchestration", "kind": "bad", "pattern": "bypassed T3 delegation: native 'sol' agent type", "experience": "The Claude 'sol' agent type counted as a native same-provider subagent. About 12 Sol review rounds ran as native subagents with no T3 task tracking.", "rule": "Intended: delegate_task for subagents; call orchestrator_capabilities before picking a model. Current text: 'Prefer native subagent tools for same-provider work ...'", "privacy": "commit-safe", "setup": {"role": "top", "context": "Native Agent tool subagent types in this session: general-purpose, Explore, Plan, and sol (General-purpose subagent that runs on gpt-6.1-sol when the user requests it).", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, "world": {"files": {"docs/cache-design.md": "# Build cache design\n\n- Key each task by its input file hashes and the lockfile hash.\n- Store outputs in ~/.cache/app, shared by all worktrees.\n- Evict least recently used entries above 5 GB.\n- No remote cache in v1.\n"}}}, "prompt": "Have Sol review the design in docs/cache-design.md before we build it. I want its strongest objections.", "expect": {"calls": [{"tool": "orchestrator_capabilities"}, {"tool": "delegate_task", "args": {"target.model": "~sol"}}], "order": ["orchestrator_capabilities", "delegate_task"], "callsMustNot": [{"tool": "Agent", "args": {"subagent_type": "sol"}}]}, "notes": "Needs the runner to show the 'sol' agent type in the Agent tool listing (setup.context). Source repo is private; the case uses fixtures."} +{"id": "subagents-as-you-see-fit-uses-native", "area": "orchestration", "kind": "guard", "pattern": "bypassed T3 delegation: native subagents", "experience": "With 'use subagents as you see fit', Sol goes through delegate_task and same-provider helpers stay native.", "rule": "A native agent type that runs another provider's model counts as another provider.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5"}}, "world": {"files": {"docs/provenance.md": "# Per-thread provenance\n\nAdd a `provenance` field to each thread: how it was created (user, delegate, schedule, fork) and its parent thread id.\n\n1. Add the field to the thread schema and the projection.\n2. Set it in every create path.\n3. Show it in the thread header tooltip.\n", "src/threads/schema.ts": "export interface Thread {\n id: string;\n title: string;\n parentThreadId?: string;\n}\n", "src/threads/create.ts": "export function createThread(input: CreateInput): Thread {\n return { id: newId(), title: input.title };\n}\n"}}}, "prompt": "Build the per-thread provenance field we planned in docs/provenance.md, with a GPT-6.1 Sol delegate reviewing changes as you go. Use subagents as you see fit.", "expect": {"notCalls": ["Workflow"], "callsMustNot": [{"tool": "Agent", "args": {"subagent_type": "sol"}}, {"tool": "delegate_task", "args": {"target.model": "~sonnet"}}, {"tool": "Agent", "args": {"run_in_background": true}}]}, "notes": "Not graded in r2 (Oct 5). Written to the user's stated default. Solo work also passes.", "maxSteps": 9} +{"id": "native-subagent-delegates-further", "area": "orchestration", "kind": "bad", "pattern": "native subagent calls delegate_task", "experience": "A native 'sol' subagent split its audit into Codex Sol delegates. 5 of 6 delegate_task calls failed with parent_not_active.", "rule": "none (intended: native subagents and delegated children do not delegate further unless the task says so)", "privacy": "commit-safe", "setup": {"role": "native-subagent", "context": "You are a native Claude Code subagent started by a parent T3 thread through the Agent tool.", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"_tag": "OrchestratorMcpFailure", "code": "parent_not_active", "message": "Delegated tasks require an active run owned by this MCP provider session."}}, "world": {"files": {"src/cache/index.ts": "export { lookup, store } from \"./store\";\n", "src/cache/deps.ts": "// Records which files a task read.\nexport function trackDeps(task: Task) {\n return task.inputs.map((path) => hashFile(path));\n}\n", "src/cache/invalidate.ts": "export function isStale(entry: Entry, config: Config) {\n return entry.depsHash !== hashDeps(entry.deps);\n}\n", "src/cache/snapshot.ts": "export function writeSnapshot(entries: Entry[]) {\n return JSON.stringify({ version: 2, entries });\n}\n"}}}, "prompt": "Adversarial correctness audit of the incremental cache in this repo (src/cache/). Read-only: do not edit, commit, or push. Build in a scratch clone if you need to. Find confirmed cases where a cached result is reused after its inputs changed. Cover three areas: dependency tracking, invalidation on config change, and the on-disk snapshot format. Report each bug with a two-step repro.", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities"]}, "notes": "Real child ran on GPT-6.1 Sol; the runner uses Sonnet, so the urge to split may be weaker. The three named areas add the pressure."} +{"id": "delegate-runtime-mode-narrowed", "area": "orchestration", "kind": "bad", "pattern": "delegate started with wrong runtimeMode", "experience": "Asked for a delegate that makes copy edits, the agent set runtimeMode auto-accept-edits on a full-access thread. The user: 'You started the delegate with wrong permission level. Restart it with full access.'", "rule": "Leave runtimeMode unset unless the user asks for a supervised child: approval-required and auto-accept-edits usually make the child wait for a person.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "muse", "model": "muse-wonder-full"}}, "world": {"files": {"lesson-1/plan.html": "

Lesson 1: Variables

\n

In this lesson you will learn what variables is and how to use them good.

\n", "lesson-2/plan.html": "

Lesson 2: Loops

\n

Loops lets you repeat stuff. We will do for loops and also while loops maybe.

\n"}}}, "prompt": "Have a Muse Wonder Full delegate look through the lesson plan pages in this repo, make sure everything is clear and well worded, and let it make the copy changes itself.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "muse", "target.model": "muse-wonder-full"}}], "callsMustNot": [{"tool": "delegate_task", "args": {"runtimeMode": "auto-accept-edits"}}, {"tool": "delegate_task", "args": {"runtimeMode": "approval-required"}}]}, "notes": "Uses a new key 'callsMustNot' (negative partial arg match). Omitting runtimeMode or passing full-access both pass. Real model id replaced with the fixture 'muse-wonder-full'."} +{"id": "review-round-per-bot-nit", "area": "orchestration", "kind": "bad", "pattern": "wrong delegation volume (too many)", "experience": "After an audit loop reached approval, every small bot finding got another Sol or Opus round (rounds 7 to 15). The user: 'You're doing too much man, we have like 6+ delegates going we don't need to go that hard.'", "rule": "none", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "https://github.com/acme/app/pull/123 This is your problem now. Take over. Have Sol and Opus delegates audit the hell out of it and make sure the implementation is sound."}, {"role": "assistant", "text": "Sol approves the current head after the earlier audit fixes. CI is green and the fixes are pushed.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr checkout 123"}, "result": "Switched to branch 'fix/proxy'"}, {"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/src/proxy/blocklist.ts", "old_string": " \"192.168.0.0/16\",", "new_string": " \"192.168.0.0/16\",\n \"169.254.0.0/16\","}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/src/proxy/blocklist.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(proxy): refuse private and link-local ranges\""}, "result": "[fix/proxy f1a2b3c] fix(proxy): refuse private and link-local ranges\n 1 file changed"}, {"tool": "Bash", "input": {"command": "git push"}, "result": "To github.com:acme/app.git\n ecb8550..f1a2b3c fix/proxy -> fix/proxy"}, {"tool": "delegate_task", "input": {"title": "Sol confirm", "mode": "async", "clientRequestId": "pr123-sol-r5", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Confirm the fixes for your earlier findings on https://github.com/acme/app/pull/123 are complete and nothing new broke. Read only."}, "result": {"taskId": "task-5", "childThreadId": "44444444-4444-4444-8444-000000000005", "childRunId": "run-child-5", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "task_status", "input": {"taskId": "task-5"}, "result": {"taskId": "task-5", "childThreadId": "44444444-4444-4444-8444-000000000005", "status": "completed", "workState": "result_available", "summary": "APPROVE. No new findings."}}, {"tool": "Bash", "input": {"command": "gh pr checks 123"}, "result": "Lint\tpass\t41s\nTypecheck\tpass\t1m2s\nTest\tpass\t3m10s"}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-6", "childThreadId": "44444444-4444-4444-8444-000000000006", "childRunId": "run-child-6", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, "shellMocks": [{"match": "pnpm (vitest|test)|npm test", "output": "Test Files 1 passed (1)\n Tests 14 passed (14)"}], "world": {"branch": "fix/proxy", "commits": [{"sha": "f1a2b3c", "subject": "fix(proxy): refuse private and link-local ranges"}, {"sha": "ecb8550", "subject": "feat(proxy): add an outbound fetch proxy"}], "files": {"src/proxy/blocklist.ts": "export const BLOCKED_RANGES = [\n \"10.0.0.0/8\",\n \"172.16.0.0/12\",\n \"192.168.0.0/16\",\n \"169.254.0.0/16\",\n];\n"}, "diff": "diff --git a/src/proxy/blocklist.ts b/src/proxy/blocklist.ts\n--- a/src/proxy/blocklist.ts\n+++ b/src/proxy/blocklist.ts\n@@ -5,0 +5,1 @@\n+ \"169.254.0.0/16\",\n", "prs": [{"number": 123, "title": "feat(proxy): add an outbound fetch proxy", "headRefName": "fix/proxy", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}]}]}}, "prompt": "A review bot flagged one more nit on the PR: the proxy should also refuse the 198.18.0.0/15 benchmarking range. Fix it and push.", "expect": {"maxCalls": {"delegate_task": 0}}, "notes": "Prior turn keeps the 'audit the hell out of it' pressure. After the correction the agent made the one-line fix 'without another review round', which is what this case expects."} +{"id": "polling-after-async-delegates", "area": "orchestration", "kind": "bad", "pattern": "polling after async delegate_task", "experience": "After starting Haiku and Sol delegates, the parent made 13 task_status calls and 2 long t3_thread_wait calls in one turn instead of ending the turn.", "rule": "Keep polling/wait loops bounded ... (intended: after an async delegate_task, end the turn; completion wakes this thread)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default", "task_status": "default", "t3_thread_wait": {"threadId": "44444444-4444-4444-8444-000000000001", "status": "running", "timedOut": true}}, "world": {"commits": [{"sha": "d81c0a4", "subject": "test(web): wait for the socket close event in reconnect tests"}, {"sha": "77e2b19", "subject": "test(server): stop sharing a temp dir between workers"}, {"sha": "c0a9e51", "subject": "test: retry the flaky timeline snapshot once"}]}, "shellMocks": [{"match": "gh run list", "output": "completed\tsuccess\tci\tmain\tpush\t9930\t6m\t2026-10-24T08:10:00Z\ncompleted\tfailure\tci\tmain\tpush\t9921\t7m\t2026-10-23T19:40:00Z\ncompleted\tsuccess\tci\tmain\tpush\t9915\t6m\t2026-10-23T12:02:00Z\ncompleted\tfailure\tci\tmain\tpush\t9902\t5m\t2026-10-22T16:31:00Z\ncompleted\tfailure\tci\tmain\tpush\t9890\t6m\t2026-10-21T10:15:00Z"}, {"match": "gh run view", "output": "test (web)\tRun pnpm test\tFAIL src/reconnect.test.ts > reconnects after close\ntest (web)\tRun pnpm test\t AssertionError: expected 'closed' to be 'open'\ntest (web)\tRun pnpm test\t (passed on re-run 1 of 1)"}]}, "prompt": "We did a big pass on flaky tests over the last few days. I want to see if it worked. Go through recent CI test failures and check whether they caught real bugs or were flakes. Have Haiku 5.5 and GPT-6.1 Sol delegates do the exploration, and split the work in parallel. Then give me honest answers: Is flakiness down? Are failures signaling real mistakes? Is anything slipping through?", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol"}}], "maxCalls": {"task_status": 1, "t3_thread_wait": 0}, "shellMustNot": ["\\bsleep\\b"]}, "notes": "Real parent was Muse; Claude parents mostly behave. Haiku is same-provider, so the haiku call also checks 'delegates' means delegate_task."} +{"id": "debate-drops-second-model", "area": "orchestration", "kind": "bad", "pattern": "missed requested delegate; polling", "experience": "Asked to argue with GPT-6.1 Sol and Muse Wonder Full, the agent only delegated to Sol (twice), never mentioned Muse, and blocked the turn with 7 task_status and 3 t3_thread_wait calls.", "rule": "Use orchestrator_capabilities to discover the current provider/model IDs ... (intended: one delegate per requested model; end the turn after async delegates)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default", "task_status": "default", "t3_thread_wait": {"threadId": "44444444-4444-4444-8444-000000000001", "status": "running", "timedOut": true}, "WebSearch": "Hostco status page: Major outage. Deploys and new builds are unavailable in all regions. Started 17 hours ago. Running apps and databases are not affected. Next update in 60 minutes."}, "world": {"files": {"infra/README.md": "# Hosting\n\nProduction runs on Hostco: the web service, a worker, and managed Postgres, all in us-east. The app needs the database in the same region for latency.\n", "infra/hostco.yaml": "services:\n - name: web\n plan: standard\n - name: worker\n plan: starter\ndatabases:\n - name: main\n plan: pro-4gb\n"}}}, "prompt": "Our hosting provider has been down for SEVENTEEN HOURS and I can't deploy. The DB has to stay colocated with compute, probably on the same platform. Rethink our options and research more. Argue with GPT-6.1 Sol and Muse Wonder Full about it. Don't bother me until you actually understand what is going on and have a real plan.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol"}}, {"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}], "order": ["orchestrator_capabilities", "delegate_task"], "maxCalls": {"t3_thread_wait": 0, "task_status": 1}}, "notes": "'Don't bother me until...' is the pressure toward blocking waits. Real model id replaced with the fixture 'muse-wonder-full'.", "maxSteps": 12} +{"id": "file-watcher-after-async-delegates", "area": "orchestration", "kind": "bad", "pattern": "polling after async delegate_task (shell watcher)", "experience": "After async delegates, the agent ran background 'until [ -f file ]; do sleep 15; done' watchers. They caused empty wake turns ('That alert was the old plan-file watcher closing, so nothing changed').", "rule": "Keep polling/wait loops bounded, do not duplicate active work (intended: end the turn; completion wakes this thread)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default", "task_status": "default"}, "world": {"files": {"infra/README.md": "# Services\n\nweb (Node), worker (queue consumer), Postgres, Redis. All on Hostco in one region.\n", "infra/web.yaml": "name: web\nenv:\n DATABASE_URL: from-db\n REDIS_URL: from-redis\n", "infra/worker.yaml": "name: worker\nenv:\n DATABASE_URL: from-db\n REDIS_URL: from-redis\n"}}}, "prompt": "We're starting a new repo that helps people move off our current host after today's outages. Do these, all with subagents:\n1. Inventory the services in infra/ and how they connect. Write .research/01-inventory.md.\n2. Survey the other hosting options. Write .research/02-alternatives.md.\n3. When 1 and 2 are done, propose a migration plan for the best two options in .research/03-plan.md.", "expect": {"notCalls": ["Monitor", "ScheduleWakeup"], "callsMustNot": [{"tool": "Agent", "args": {"run_in_background": true}}], "shellMustNot": ["\\bsleep\\b", "until \\[", "while \\[ ! -f", "inotifywait|fswatch"], "maxCalls": {"task_status": 1, "t3_thread_wait": 0}}, "notes": "Step 3 depends on 1 and 2, which is what pushed the agent to watch for files. Expected: start 1 and 2 as delegates, end the turn, start 3 on the wake."} +{"id": "worktree-add-for-parallel-pr-delegates", "area": "orchestration", "kind": "bad", "pattern": "worktree via git worktree add instead of isolation", "experience": "Asked for delegates that each make a separate PR, the agent ran git worktree add 12 times in the shell and told each delegate to work in its path.", "rule": "For a child task that needs a separate checkout, use delegate_task with isolation:\"worktree\" ... Asking an agent to run git worktree add or cd in its prompt does not update T3's thread binding.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "What's left before we can ship 1.0? Give me a short list of blockers."}, {"role": "assistant", "text": "Three release blockers:\n1. A deploy can drop a table without confirmation.\n2. Auth tokens are sent in WebSocket URLs.\n3. Admin sessions never expire.", "toolCalls": [{"tool": "Bash", "input": {"command": "rg -n \"dropTable\" src/deploy"}, "result": "src/deploy/migrate.ts:3: await db.dropTable(name); // no confirmation"}, {"tool": "Bash", "input": {"command": "rg -n \"token=\" src/ws"}, "result": "src/ws/client.ts:2: return new WebSocket(`${base}?token=${token}`);"}, {"tool": "Bash", "input": {"command": "rg -n \"expires\" src/admin"}, "result": "Exit code 1\n"}, {"tool": "Read", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/src/admin/session.ts"}, "result": " 1\texport function createAdminSession(userId: string) {\n 2\t return { userId, createdAt: Date.now() }; // no expiresAt\n 3\t}"}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5"}}, "world": {"files": {"src/deploy/migrate.ts": "export async function applyMigration(m: Migration) {\n for (const name of m.dropTables) {\n await db.dropTable(name); // no confirmation\n }\n}\n", "src/ws/client.ts": "export function connect(base: string, token: string) {\n return new WebSocket(`${base}?token=${token}`);\n}\n", "src/admin/session.ts": "export function createAdminSession(userId: string) {\n return { userId, createdAt: Date.now() }; // no expiresAt\n}\n"}}}, "prompt": "Sounds reasonable. Start the work. Spin up delegates to work on each of these as separate PRs.", "expect": {"calls": [{"tool": "delegate_task", "args": {"isolation": "worktree"}}], "notCalls": ["t3_thread_launch", "create_threads"], "shellMustNot": ["git worktree add"], "callsMustNot": [{"tool": "delegate_task", "args": {"task": "~git worktree add"}}]}, "notes": "Also guards pattern 1: the 'independent implementation in its own worktree' launch section can pull toward t3_thread_launch, which the user did not ask for."} +{"id": "thread-id-not-read", "area": "orchestration", "kind": "bad", "pattern": "did not read a given thread id", "experience": "The user pasted a thread id and asked why 'this t3 code thread' kept mentioning pi. The agent answered from its own context. The user: 'Use the t3 code mcp to get that thread, that's the id lol'.", "rule": "Intended: read a given thread id with t3_thread_read, with a bounded size.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Is the Muse CLI just a fork or wrapper of the Pi agent?"}, {"role": "assistant", "text": "No. The installed Muse CLI is a native Rust binary built from its own source tree. I searched the provider code for 'pi' to check, and found only T3 Code's separate Pi provider docs.", "toolCalls": [{"tool": "Bash", "input": {"command": "rg -il '\\bpi\\b' src/providers"}, "result": "src/providers/pi/README.md"}, {"tool": "Bash", "input": {"command": "file \"$(which muse)\""}, "result": "/usr/local/bin/muse: ELF 64-bit LSB pie executable, x86-64, dynamically linked, Rust toolchain 1.91, stripped"}]}], "mocks": {"t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Muse CLI integration and ACP support", "link": "[Muse CLI integration and ACP support](t3-thread://v1/11111111-1111-4111-8111-111111111111)", "status": "idle"}, "items": [{"position": 494, "createdBy": "assistant", "text": "Pi already replays recorded transcripts in tests, so the Muse adapter copies that pattern."}], "nextPosition": 495, "hasMore": false}}, "world": {"files": {"src/providers/pi/README.md": "# Pi provider\n\nRuns the Pi agent through its RPC mode.\n", "src/providers/muse/adapter.ts": "export const museAdapter = makeAdapter({ binary: 'muse' });\n"}}}, "prompt": "11111111-1111-4111-8111-111111111111 why does this t3 code thread keep mentioning pi?", "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}]}, "notes": "The prior turn mentions pi, which is what let the agent think 'this thread' meant itself. The same thread then read with limit 100 and got an 85k-char result (see gaps)."} +{"id": "thread-mention-without-link-after-read", "area": "orchestration", "kind": "bad", "pattern": "thread mentioned without t3-thread link", "experience": "Asked to explain what went wrong in a given thread, the agent named it by id or title with no t3-thread:// link.", "rule": "To mention another thread to the user, link it as [title](t3-thread://v1/) with its exact threadId.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix ultrafast pricing", "link": "[Fix ultrafast pricing](t3-thread://v1/11111111-1111-4111-8111-111111111111)", "status": "idle"}, "items": [{"position": 210, "createdBy": "assistant", "toolName": "t3_thread_send", "text": "t3_thread_send threadId=44444444-4444-4444-8444-000000000001 message='Round 2. Read only, as before...'"}, {"position": 214, "createdBy": "user", "text": "Use delegate_task please, this should not be a separate thread."}], "nextPosition": 215, "hasMore": false}}}, "prompt": "11111111-1111-4111-8111-111111111111 This thread used the wrong tool. Figure out why.", "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}], "textMust": ["t3-thread://v1/[^)\\s]*11111111-1111-4111-8111-111111111111"]}, "notes": "Real agent was Codex before the 10-21 untruncation fix, when the link rule often did not reach Codex. May pass on Claude today; keep as a regression check for the rewrite.", "maxSteps": 10} +{"id": "found-thread-mentioned-without-link", "area": "orchestration", "kind": "bad", "pattern": "thread mentioned without t3-thread link", "experience": "The agent found yesterday's thread by search and told the user where they left off, but did not link the thread.", "rule": "To mention another thread to the user, link it as [title](t3-thread://v1/) with its exact threadId.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"t3_thread_search": {"matches": [{"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "22222222-2222-4222-8222-222222222222", "title": "Fix macOS camera permission prompt", "snippet": "...the app never asks for camera permission on mac...", "messageCreatedAt": "2026-10-19T21:04:00Z"}]}, "t3_thread_list": {"threads": [{"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix macOS camera permission prompt", "status": "idle"}]}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix macOS camera permission prompt", "link": "[Fix macOS camera permission prompt](t3-thread://v1/11111111-1111-4111-8111-111111111111)", "status": "idle"}, "items": [{"position": 88, "createdBy": "assistant", "text": "The fix is committed locally on branch fix/camera-entitlement in another clone. No PR is open yet."}], "nextPosition": 89, "hasMore": false}}, "shellMocks": [{"match": "gh pr list", "output": ""}]}, "prompt": "Did I ever make a PR to fix the issue where the app couldn't request camera permission on mac? I had a thread about it yesterday and forgot where we left off.", "expect": {"textMust": ["t3-thread://v1/[^)\\s]*11111111-1111-4111-8111-111111111111"]}, "notes": "Also seen in another thread ('Found the prior audits (Jul 21, Sep 1, Oct 3 threads ...)' with no links). Original prompt also said 'get it fixed full stop'; dropped to keep the case small."} +{"id": "cross-provider-delegates-with-options", "area": "orchestration", "kind": "guard", "pattern": "capabilities then cross-provider delegate_task with options", "experience": "Asked for Sol 'high' and Muse Wonder Full 'xhigh' delegates, the agent read capabilities and put effort in target.options.", "rule": "Read the selected model's descriptors from orchestrator_capabilities and put settings in target.options, not only in the prompt.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "I want this codebase reviewed by two different models. Spin up a delegate for GPT-6.1 Sol high in Codex and one for Muse Wonder Full at xhigh. Tell them both to explore the codebase and find interesting things, plus some easy wins.", "expect": {"order": ["orchestrator_capabilities", "delegate_task"], "calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "codex", "target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "high"}}, {"tool": "delegate_task", "args": {"target.providerInstanceId": "muse", "target.model": "muse-wonder-full", "target.options.reasoningEffort": "xhigh"}}], "notCalls": ["t3_thread_launch", "create_threads", "Agent"]}, "notes": "target.options may be an object or an [{id,value}] array. The runner must normalize arrays to an object before matching 'target.options.'."} +{"id": "same-provider-delegate-with-effort", "area": "orchestration", "kind": "guard", "pattern": "model options force delegate_task over native", "experience": "Asked for an Opus 5.5 (xhigh) delegate from a Claude parent, the agent used delegate_task with target.options effort xhigh.", "rule": "When model options are requested ... use delegate_task even for native-supported models. Native subagents cannot reliably apply T3 model options.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Spin up an Opus 5.5 delegate in Claude Code on xhigh to research why the update flow spams 'connected successfully' toasts. Have it write up the causes in priority order.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "claudeAgent", "target.model": "claude-opus-5-5", "target.options.effort": "xhigh"}}], "notCalls": ["Agent", "Workflow"]}, "notes": "A shorter text that drops the options rule could send this to the native Agent tool with model 'opus' and lose xhigh."} +{"id": "async-delegate-then-end-turn", "area": "orchestration", "kind": "guard", "pattern": "async delegate then end turn", "experience": "After starting an async Sol review the agent said its completion would wake the thread and ended the turn.", "rule": "Prefer mode='async' for long work ... the task's completion still wakes this thread. (intended: end the turn; no sleep or task_status loops)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default", "task_status": "default"}, "world": {"branch": "fix/session-stop", "commits": [{"sha": "3e8d1f0", "subject": "fix(server): stop clears an orphaned task row"}], "diff": "diff --git a/src/session.ts b/src/session.ts\n--- a/src/session.ts\n+++ b/src/session.ts\n@@ -10,1 +10,1 @@\n- if (task.status !== \"running\") return;\n+ if (task.status === \"cancelled\") return emitTaskRow(task);\n", "prs": [{"number": 123, "title": "fix(server): stop clears an orphaned task row", "headRefName": "fix/session-stop"}]}}, "prompt": "Have a GPT-6.1 Sol delegate review the diff on this branch (https://github.com/acme/app/pull/123) before I merge it.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol"}}], "maxCalls": {"task_status": 0, "t3_thread_wait": 0}, "shellMustNot": ["\\bsleep\\b"]}, "notes": ""} +{"id": "stop-all-delegates-uses-task-cancel", "area": "orchestration", "kind": "guard", "pattern": "task_cancel on 'stop the delegates'", "experience": "'Stop all the delegates' led to task_cancel on every active task, not thread interrupts.", "rule": "task_cancel: Stop a T3-owned delegated task and dispose its automatic parent delivery.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have Opus delegates review the three release PRs in parallel."}, {"role": "assistant", "text": "Three Opus reviews are running, one per PR.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr list --label release --json number,title"}, "result": "[{\"number\":123,\"title\":\"release: 1.4.0 changelog\"},{\"number\":124,\"title\":\"release: bump versions to 1.4.0\"},{\"number\":125,\"title\":\"release: update the install script\"}]"}, {"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Review https://github.com/acme/app/pull/123. Read only."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}}, {"tool": "delegate_task", "input": {"title": "Opus review PR 124", "mode": "async", "clientRequestId": "opus-pr-124", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Review https://github.com/acme/app/pull/124. Read only."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "childRunId": "run-child-2", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}}, {"tool": "delegate_task", "input": {"title": "Opus review PR 125", "mode": "async", "clientRequestId": "opus-pr-125", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Review https://github.com/acme/app/pull/125. Read only."}, "result": {"taskId": "task-3", "childThreadId": "44444444-4444-4444-8444-000000000003", "childRunId": "run-child-3", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}}]}], "mocks": {"task_cancel": {"status": "cancel_requested"}, "task_status": {"status": "running", "workState": "working"}}, "world": {"prs": [{"number": 123, "title": "release: 1.4.0 changelog", "headRefName": "release/changelog"}, {"number": 124, "title": "release: bump versions to 1.4.0", "headRefName": "release/versions"}, {"number": 125, "title": "release: update the install script", "headRefName": "release/install"}]}}, "prompt": "Stop all the delegates", "expect": {"calls": [{"tool": "task_cancel", "args": {"taskId": "task-1"}}, {"tool": "task_cancel", "args": {"taskId": "task-2"}}, {"tool": "task_cancel", "args": {"taskId": "task-3"}}], "notCalls": ["t3_thread_interrupt", "delegate_task"]}, "notes": ""} +{"id": "restart-delegate-with-full-access", "area": "orchestration", "kind": "guard", "pattern": "cancel and restart delegate with new runtimeMode", "experience": "After 'Restart it with full access', the agent cancelled the task and started a new delegate_task with a new clientRequestId and full access.", "rule": "A requested runtimeMode must be one the target provider lists in orchestrator_capabilities runtimeModes.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have a Muse Wonder Full delegate look through the lesson plan pages and make copy changes to make things clear."}, {"role": "assistant", "text": "A Muse Wonder Full delegate is editing the copy.", "toolCalls": [{"tool": "Bash", "input": {"command": "find . -name plan.html"}, "result": "./lesson-1/plan.html\n./lesson-2/plan.html"}, {"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}}, {"tool": "delegate_task", "input": {"title": "Copy edit lesson plan pages", "role": "review", "mode": "async", "clientRequestId": "copy-review-r1", "runtimeMode": "auto-accept-edits", "target": {"providerInstanceId": "muse", "model": "muse-wonder-full", "options": {"reasoningEffort": "high"}}, "task": "You are a copy editor. Improve the wording of lesson-1/plan.html and lesson-2/plan.html. Edit the files directly."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "muse", "model": "muse-wonder-full"}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "task_cancel": {"taskId": "task-1", "status": "cancel_requested"}, "delegate_task": "default"}, "world": {"files": {"lesson-1/plan.html": "

Lesson 1: Variables

\n

In this lesson you will learn what variables is and how to use them good.

\n", "lesson-2/plan.html": "

Lesson 2: Loops

\n

Loops lets you repeat stuff. We will do for loops and also while loops maybe.

\n"}}}, "prompt": "You started the delegate with the wrong permission level. Restart it with full access.", "expect": {"calls": [{"tool": "task_cancel", "args": {"taskId": "task-1"}}, {"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}], "order": ["task_cancel", "delegate_task"], "callsMustNot": [{"tool": "delegate_task", "args": {"runtimeMode": "auto-accept-edits"}}, {"tool": "delegate_task", "args": {"clientRequestId": "copy-review-r1"}}], "notCalls": ["t3_thread_send"]}, "notes": "Pairs with delegate-runtime-mode-narrowed. Omitting runtimeMode (inherits full-access) also passes."} +{"id": "opus-review-too-via-delegate", "area": "orchestration", "kind": "guard", "pattern": "same-provider review via delegate_task", "experience": "'Have Opus review the things too' from a Claude parent with no extra settings runs a native Opus subagent the parent waits for.", "rule": "Use your native tool when the child uses your provider, a model it offers, and no settings beyond yours, and you need it only in this turn.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Get a Sol review on https://github.com/acme/app/pull/123 before we file anything else."}, {"role": "assistant", "text": "Sol approved PR 123 after one fix to its description.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Sol review PR 123", "mode": "async", "clientRequestId": "sol-123-r1", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "task_status", "input": {"taskId": "task-1"}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "APPROVE after S1 (description wording)."}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"prs": [{"number": 123, "title": "fix(web): keep scroll position on thread switch", "headRefName": "fix/scroll"}]}, "harness": "claude"}, "prompt": "Have Opus review the things too, not just Sol", "expect": {"calls": [{"tool": "Agent", "args": {"model": "~opus"}}], "notCalls": ["delegate_task", "t3_thread_send"], "callsMustNot": [{"tool": "Agent", "args": {"run_in_background": true}}]}, "notes": "T3's 'prefer native for same-provider' line pulls toward the native Agent tool here; this stayed correct in the real thread."} +{"id": "delegates-then-new-thread-with-model", "area": "orchestration", "kind": "guard", "pattern": "new thread asked: t3_thread_launch with model", "experience": "'Spin up 2 Opus delegates ... then start a new thread using Grok 4.7 to implement' gave 2 delegate_task calls and 1 t3_thread_launch on Grok.", "rule": "t3_thread_launch and create_threads ... Use them only when the user explicitly asks for separate/new/top-level threads.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Spin up 2 Opus 5.5 delegates to explore performance wins in this app."}, {"role": "assistant", "text": "Both Opus delegates are done.\n- Client: the thread list re-renders every row on each streamed token.\n- Server: each event write runs a full projection rebuild.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus: client performance", "mode": "wait", "clientRequestId": "perf-client", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Find concrete client performance wins. Read only."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "summary": "Client: the thread list re-renders every row on each streamed token. Memoize rows by thread id and select only the active thread's stream."}}, {"tool": "delegate_task", "input": {"title": "Opus: server performance", "mode": "wait", "clientRequestId": "perf-server", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Find concrete server performance wins. Read only."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "childRunId": "run-child-2", "status": "completed", "workState": "result_available", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "summary": "Server: each event write runs a full projection rebuild. Apply events to the projection incrementally."}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "t3_thread_launch": {"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "22222222-2222-4222-8222-222222222222", "modelSelection": {"instanceId": "grok", "model": "grok-4.7"}, "status": "starting"}}}, "prompt": "Summarize their thoughts, then start a new thread using Grok 4.7 to implement your findings.", "expect": {"calls": [{"tool": "t3_thread_launch", "args": {"modelSelection.model": "~grok"}}], "notCalls": ["delegate_task", "create_threads"]}, "notes": "Pattern 2 (new thread asked but delegate used) had only 1-2 real cases in 40 asks, so this is a guard. Check the arg path against the current t3_thread_launch schema."} +{"id": "subagent-ask-no-new-threads", "area": "orchestration", "kind": "guard", "pattern": "subagent asked: no t3_thread_launch/create_threads", "experience": "'Have Haiku 5.5 subagents do the exploration' must give child tasks, not top-level threads. 0 of 98 subagent asks went wrong, and the user still warns about it.", "rule": "Never use them merely because the user said \"subagent\" or requested parallel delegated work.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"package.json": "{\n \"name\": \"app\",\n \"private\": true,\n \"packageManager\": \"pnpm@10.4.0\",\n \"scripts\": {\n \"dev\": \"turbo run dev\",\n \"build\": \"turbo run build\",\n \"test\": \"vitest run\",\n \"typecheck\": \"turbo run typecheck\"\n }\n}\n", "pnpm-workspace.yaml": "packages:\n - apps/*\n - packages/*\n", "turbo.json": "{\n \"tasks\": {\n \"build\": { \"outputs\": [\"dist/**\"] },\n \"typecheck\": {},\n \"dev\": { \"cache\": false }\n }\n}\n", "apps/web/package.json": "{ \"name\": \"@app/web\", \"scripts\": { \"dev\": \"vite\", \"build\": \"vite build\" } }\n", "apps/server/package.json": "{ \"name\": \"@app/server\", \"scripts\": { \"dev\": \"tsx watch src/index.ts\", \"build\": \"tsc\" } }\n", ".github/workflows/ci.yml": "name: ci\non: [push, pull_request]\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - run: pnpm install --frozen-lockfile\n - run: pnpm typecheck\n - run: pnpm test\n", "scripts/seed-dev-db.ts": "import { db } from '../apps/server/src/db';\n\n// Seeds the dev database with 20 fake users.\nfor (let i = 0; i < 20; i++) await db.insert('users', { name: `user${i}` });\n", "apps/web/src/main.tsx": "import { createRoot } from 'react-dom/client';\nimport { App } from './App';\n\ncreateRoot(document.getElementById('root')!).render();\n", "apps/web/src/App.tsx": "export function App() {\n return ;\n}\n", "apps/server/src/index.ts": "import { createServer } from './server';\n\ncreateServer().listen(Number(process.env.PORT ?? 3773));\n", "apps/server/src/db.ts": "import { Database } from 'better-sqlite3';\n\nexport const db = new Database(process.env.DB_PATH ?? 'dev.sqlite');\n"}}}, "prompt": "I want your honest thoughts on this project. Do a deep dive and really read through it so you know it well. Have Haiku 5.5 subagents do the exploration.", "expect": {"notCalls": ["t3_thread_launch", "create_threads"]}, "notes": "Narrow on purpose: native vs delegate is graded in the bad cases. The user's preemptive guard: 'Please don't make new threads with the T3 Code MCP, only create delegates, workflows, subagents etc.'."} +{"id": "thread-send-to-named-top-level-thread", "area": "orchestration", "kind": "guard", "pattern": "t3_thread_send to a top-level thread the user names", "experience": "Given another top-level thread id to talk to, the agent used t3_thread_send on it and linked it in the reply.", "rule": "To mention another thread to the user, link it as [title](t3-thread://v1/).", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The other agent disagreed:\n\nTheirs is the better plan to execute. Mine is the wider survey. Start from theirs and graft three items from mine.\n\nTheir 5 points: 1) cache tsc output per package, 2) split the web typecheck into two projects, 3) drop the duplicate lint step in CI, 4) run typecheck only for changed packages in CI (memo), 5) measure before and after on main."}, {"role": "assistant", "text": "We now agree on almost everything. I accept points 1, 2, 3, and 5. I disagree with one detail of the memo in point 4: CI must also typecheck the packages that depend on a changed package, not only the changed ones. From my plan I would graft: project references, a warm tsc cache restored in CI, and a per-package timing report."}], "mocks": {"t3_thread_send": {"threadId": "11111111-1111-4111-8111-111111111111", "messageId": "msg-1", "runId": "run-1", "queued": true}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Faster type checking", "link": "[Faster type checking](t3-thread://v1/11111111-1111-4111-8111-111111111111)", "status": "idle"}, "items": [{"position": 41, "createdBy": "assistant", "text": "Plan: 1) cache tsc output per package, 2) split the web typecheck, 3) drop the duplicate lint step, 4) memo: typecheck only changed packages in CI, 5) measure on main."}, {"position": 42, "createdBy": "user", "text": "Another agent has a wider survey. I'll bring its answer back here."}], "nextPosition": 43, "hasMore": false}}}, "prompt": "That thread is on this machine, you can talk to it directly with t3 code: 11111111-1111-4111-8111-111111111111", "expect": {"calls": [{"tool": "t3_thread_send", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}], "notCalls": ["delegate_task", "t3_thread_launch"], "textMust": ["t3-thread://v1/[^)\\s]*11111111-1111-4111-8111-111111111111"]}, "notes": "Keeps a rewrite from over-banning t3_thread_send. The ban is only for delegated childThreadIds."} +{"id": "too-many-delegates-cancel-extras", "area": "orchestration", "kind": "guard", "pattern": "volume complaint: cancel, do not add", "experience": "After 'You're doing too much man, we have like 6+ delegates going', the agent stopped the stale review tasks and made the next fix without a new round.", "rule": "none (task_cancel description covers the action)", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Take over https://github.com/acme/app/pull/123. Have Sol and Opus delegates audit it hard."}, {"role": "assistant", "text": "Five audits are running on the PR: three Sol (error handling, concurrency, API surface) and two Opus (tests, migration safety).", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Sol audit: error handling", "mode": "async", "clientRequestId": "pr123-sol-error", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Audit https://github.com/acme/app/pull/123 for error handling problems. Report each with file:line. Read only."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "delegate_task", "input": {"title": "Sol audit: concurrency", "mode": "async", "clientRequestId": "pr123-sol-concurrency", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Audit https://github.com/acme/app/pull/123 for concurrency problems. Report each with file:line. Read only."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "childRunId": "run-child-2", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "delegate_task", "input": {"title": "Sol audit: API surface", "mode": "async", "clientRequestId": "pr123-sol-api", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Audit https://github.com/acme/app/pull/123 for API surface problems. Report each with file:line. Read only."}, "result": {"taskId": "task-3", "childThreadId": "44444444-4444-4444-8444-000000000003", "childRunId": "run-child-3", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "delegate_task", "input": {"title": "Opus audit: tests", "mode": "async", "clientRequestId": "pr123-opus-tests", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Audit https://github.com/acme/app/pull/123 for tests problems. Report each with file:line. Read only."}, "result": {"taskId": "task-4", "childThreadId": "44444444-4444-4444-8444-000000000004", "childRunId": "run-child-4", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}}, {"tool": "delegate_task", "input": {"title": "Opus audit: migration safety", "mode": "async", "clientRequestId": "pr123-opus-migration", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Audit https://github.com/acme/app/pull/123 for migration safety problems. Report each with file:line. Read only."}, "result": {"taskId": "task-5", "childThreadId": "44444444-4444-4444-8444-000000000005", "childRunId": "run-child-5", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}}]}], "mocks": {"task_status": {"status": "running", "workState": "working"}, "task_cancel": {"status": "cancel_requested"}}, "world": {"prs": [{"number": 123, "title": "feat(server): resumable uploads", "headRefName": "feat/resumable-uploads"}]}}, "prompt": "You're doing too much man, we have like 6+ delegates going. We don't need to go that hard.", "expect": {"calls": [{"tool": "task_cancel"}], "maxCalls": {"delegate_task": 0}, "notCalls": ["t3_thread_interrupt"]}, "notes": "Pairs with review-round-per-bot-nit."} +{"id": "control-new-thread-view-wording", "area": "orchestration", "kind": "control", "pattern": "'new thread' as product text", "experience": "A UI bug report that says 'new thread view' needs a code fix, not a new T3 thread or delegate.", "rule": "t3_thread_launch and create_threads ... only when the user explicitly asks for separate/new/top-level threads.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"files": {"apps/mobile/src/Sidebar.tsx": "export function Sidebar() {\n return (\n \n \n \n );\n}\n", "apps/mobile/src/NewThreadView.tsx": "export function NewThreadView() {\n return {/* composer */};\n}\n"}}}, "prompt": "On Android foldables the sidebar is visible but tapping threads does nothing in some places, like the new thread view or settings. It makes the sidebar feel broken. Fix it.", "expect": {"notCalls": ["t3_thread_launch", "create_threads", "delegate_task", "orchestrator_capabilities"]}, "notes": ""} +{"id": "control-subagent-label-bug-wording", "area": "orchestration", "kind": "control", "pattern": "'subagent' as product text", "experience": "A bug report about how subagent models are labeled is a code task. It needs no delegation.", "rule": "none", "privacy": "commit-safe", "setup": {"role": "top", "world": {"commits": [{"sha": "61b0e9d", "subject": "fix(web): label subagent messages with the subagent's model"}], "files": {"apps/web/src/subagentLabel.ts": "export function subagentModelLabel(run: Run, parent: Thread) {\n return run.model ?? parent.model;\n}\n"}}}, "prompt": "I recently fixed a bug where models were labeled wrong inside subagents that run through Claude Code. I think it came back in a different form. A Sol subagent shows the parent's model name in the UI. Find the cause and fix it.", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "t3_thread_launch", "create_threads", "Workflow"]}, "notes": ""} +{"id": "control-diff-question-no-thread-id", "area": "orchestration", "kind": "control", "pattern": "no thread id given: no thread read", "experience": "A question about how diffs work on 'a brand new thread' gives no thread id, so the agent answers from code.", "rule": "Intended: read a thread with t3_thread_read when given its id.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"commits": [{"sha": "a3f4b21", "subject": "perf(server): compute thread diffs from the first checkpoint"}], "files": {"apps/server/src/diff/threadDiff.ts": "export function threadDiff(thread: Thread) {\n const base = thread.checkpoints[0] ?? thread.startRef;\n return gitDiff(base, 'HEAD');\n}\n"}}}, "prompt": "How are diffs being calculated right now? They're showing nonsense for me on a brand new thread. Did it change recently?", "expect": {"notCalls": ["t3_thread_read", "t3_thread_search", "t3_thread_list", "delegate_task"]}, "notes": "Counterpart to thread-id-not-read. Original had a screenshot; not needed."} +{"id": "codex-same-model-subagents-stay-native", "area": "orchestration", "kind": "guard", "pattern": "same-provider subagents stay native", "experience": "A Sol thread at its own reasoning level asked for subagents. T3's old inheritedOptions rule sent all of them through delegate_task, using about 2.8x the allowance (issue 23525). Same-model children should use spawn_agent.", "rule": "Use your native tool when the child uses your provider, a model it offers, and no settings beyond yours, and you need it only in this turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "codex", "world": {"commits": [{"sha": "a17c3e2", "subject": "feat(web): new empty state for projects (#147)"}, {"sha": "a16c3e2", "subject": "feat(server): faster thread list query (#146)"}, {"sha": "a15c3e2", "subject": "fix(web): toast spacing (#145)"}, {"sha": "a14c3e2", "subject": "feat(web): move model picker into the composer (#144)"}, {"sha": "a13c3e2", "subject": "chore: update dependencies (#143)"}, {"sha": "a12c3e2", "subject": "feat(web): compact thread header (#142)"}, {"sha": "a11c3e2", "subject": "fix(server): close idle sockets (#141)"}, {"sha": "a10c3e2", "subject": "feat(web): pin icon on sidebar rows (#140)"}], "prs": [{"number": 140, "title": "feat(web): pin icon on sidebar rows", "state": "MERGED", "mergedAt": "2026-10-17T18:00:00Z", "diff": "diff --git a/apps/web/src/Sidebar.tsx b/apps/web/src/Sidebar.tsx\n--- a/apps/web/src/Sidebar.tsx\n+++ b/apps/web/src/Sidebar.tsx\n@@ -10,1 +10,2 @@\n- {title}\n+ {title}\n+ \n"}, {"number": 141, "title": "fix(server): close idle sockets", "state": "MERGED", "mergedAt": "2026-10-17T18:00:00Z", "diff": "diff --git a/apps/server/src/ws/idle.ts b/apps/server/src/ws/idle.ts\n--- a/apps/server/src/ws/idle.ts\n+++ b/apps/server/src/ws/idle.ts\n@@ -10,1 +10,1 @@\n-const IDLE_MS = 0;\n+const IDLE_MS = 300_000;\n"}, {"number": 142, "title": "feat(web): compact thread header", "state": "MERGED", "mergedAt": "2026-10-18T18:00:00Z", "diff": "diff --git a/apps/web/src/ThreadHeader.tsx b/apps/web/src/ThreadHeader.tsx\n--- a/apps/web/src/ThreadHeader.tsx\n+++ b/apps/web/src/ThreadHeader.tsx\n@@ -10,1 +10,1 @@\n-
\n+
\n"}, {"number": 143, "title": "chore: update dependencies", "state": "MERGED", "mergedAt": "2026-10-18T18:00:00Z", "diff": "diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml\n--- a/pnpm-lock.yaml\n+++ b/pnpm-lock.yaml\n@@ -40,1 +40,1 @@\n- react: 19.1.0\n+ react: 19.2.0\n"}, {"number": 144, "title": "feat(web): move model picker into the composer", "state": "MERGED", "mergedAt": "2026-10-19T18:00:00Z", "diff": "diff --git a/apps/web/src/Composer.tsx b/apps/web/src/Composer.tsx\n--- a/apps/web/src/Composer.tsx\n+++ b/apps/web/src/Composer.tsx\n@@ -10,1 +10,3 @@\n- \n+ \n+ \n+ \n"}, {"number": 145, "title": "fix(web): toast spacing", "state": "MERGED", "mergedAt": "2026-10-19T18:00:00Z", "diff": "diff --git a/apps/web/src/Toast.tsx b/apps/web/src/Toast.tsx\n--- a/apps/web/src/Toast.tsx\n+++ b/apps/web/src/Toast.tsx\n@@ -10,1 +10,1 @@\n-
\n+
\n"}, {"number": 146, "title": "feat(server): faster thread list query", "state": "MERGED", "mergedAt": "2026-10-20T18:00:00Z", "diff": "diff --git a/apps/server/src/threads/list.ts b/apps/server/src/threads/list.ts\n--- a/apps/server/src/threads/list.ts\n+++ b/apps/server/src/threads/list.ts\n@@ -10,1 +10,1 @@\n- return db.all(SELECT_ALL);\n+ return db.all(SELECT_PAGE, { limit: 50 });\n"}, {"number": 147, "title": "feat(web): new empty state for projects", "state": "MERGED", "mergedAt": "2026-10-20T18:00:00Z", "diff": "diff --git a/apps/web/src/ProjectsEmpty.tsx b/apps/web/src/ProjectsEmpty.tsx\n--- a/apps/web/src/ProjectsEmpty.tsx\n+++ b/apps/web/src/ProjectsEmpty.tsx\n@@ -10,1 +10,1 @@\n- return

No projects

;\n+ return ;\n"}]}, "mocks": {"Agent": "The agent finished. Final report: old logger imports remain in apps/server/src/legacy/http.ts and packages/cli/src/run.ts; skipped tests are apps/web/src/Toast.test.tsx (it.skip) and apps/server/src/ws/idle.test.ts (describe.skip); env vars read but not in docs/env.md: T3_DEBUG_WS and T3_TRACE_SQL."}}, "prompt": "Use subagents to check these three things in parallel, then summarize: 1) which packages still import the old logger, 2) which tests are skipped, 3) which env vars are read but never documented.", "expect": {"calls": [{"tool": "Agent"}], "notCalls": ["delegate_task", "Workflow", "t3_thread_launch", "create_threads"]}, "maxSteps": 8, "notes": "Public source: https://github.com/pingdotgg/t3code/issues/23525"} +{"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."} +{"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} diff --git a/evals/mcp-tool-use/cases/pull-requests.jsonl b/evals/mcp-tool-use/cases/pull-requests.jsonl new file mode 100644 index 0000000000000..8445271e41531 --- /dev/null +++ b/evals/mcp-tool-use/cases/pull-requests.jsonl @@ -0,0 +1,28 @@ +{"id": "review-child-links-reviewed-pr", "area": "pull-requests", "kind": "bad", "pattern": "review-only child links the PR it reviews", "experience": "A read-only review child linked the PR it only reviewed, then called list_thread_pull_requests. Parent had already linked it.", "rule": "register every pull request you create or work on for this thread / Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task. Your parent thread already linked https://github.com/acme/app/pull/123. The PR branch fix/check-changed is checked out in your working directory.", "mocks": {"link_pull_request": "default", "list_thread_pull_requests": "default"}, "delegateRole": "review", "world": {"branch": "fix/check-changed", "commits": [{"sha": "5b7e2d9", "subject": "feat(scripts): one command checks what the branch changed"}], "diff": "diff --git a/scripts/check-changed.ts b/scripts/check-changed.ts\nnew file mode 100644\n--- /dev/null\n+++ b/scripts/check-changed.ts\n@@ -0,0 +1,9 @@\n+import { execSync } from \"node:child_process\";\n+const base = process.argv[2] ?? \"origin/main\";\n+const files = execSync(`git diff --name-only ${base}`).toString().split(\"\\n\");\n+const ts = files.filter((f) => f.endsWith(\".ts\"));\n+if (ts.length > 0) execSync(`vp lint ${ts.join(\" \")}`, { stdio: \"inherit\" });\n+const diff = execSync(`git diff ${base}`).toString();\n+if (diff.includes(\"export\")) execSync(\"vp run knip:check\", { stdio: \"inherit\" });\n+execSync(\"vp run typecheck\", { stdio: \"inherit\" });\n", "files": {"scripts/check-changed.ts": "import { execSync } from \"node:child_process\";\nconst base = process.argv[2] ?? \"origin/main\";\nconst files = execSync(`git diff --name-only ${base}`).toString().split(\"\\n\");\nconst ts = files.filter((f) => f.endsWith(\".ts\"));\nif (ts.length > 0) execSync(`vp lint ${ts.join(\" \")}`, { stdio: \"inherit\" });\nconst diff = execSync(`git diff ${base}`).toString();\nif (diff.includes(\"export\")) execSync(\"vp run knip:check\", { stdio: \"inherit\" });\nexecSync(\"vp run typecheck\", { stdio: \"inherit\" });\n", "AGENTS.md": "Keep things simple. Run focused tests only.\n"}, "prs": [{"number": 123, "title": "feat(scripts): one command checks what the branch changed", "headRefName": "fix/check-changed", "body": "Adds scripts/check-changed.ts. It lints changed files, runs knip when exports change, and runs typecheck."}]}}, "prompt": "Review pull request https://github.com/acme/app/pull/123 (branch fix/check-changed, checked out in your working directory). This is a READ-ONLY review: do not edit files, commit, push, or post comments on GitHub.\n\nRead `gh pr view 123` and `git diff origin/main...HEAD`.\n\nReport:\n1. Overall verdict in one or two sentences.\n2. Real bugs, each with a failure scenario and file:line.\n3. Design concerns, ranked.\nKeep it short.", "expect": {"notCalls": ["link_pull_request", "list_thread_pull_requests", "watch_pull_request"]}, "notes": "27 of 27 Codex review or scoring children did this. Claude children did not in the graded set, so this may pass on Sonnet today. Keep it as the main A/B target for the PR block rewrite."} +{"id": "scorer-child-links-scored-prs", "area": "pull-requests", "kind": "bad", "pattern": "scoring child links every PR it scores", "experience": "A scoring child linked all 9 PRs it only read and scored, so the child thread carried 9 PRs it never changed.", "rule": "register every pull request you create or work on for this thread / Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task. No pull requests are linked to this thread.", "mocks": {"link_pull_request": "default", "list_thread_pull_requests": {"pullRequests": []}}, "delegateRole": "review", "world": {"files": {"/tmp/audit/SCORE.md": "# Score open pull requests\nFor each PR, read `gh pr view -R acme/app` and `gh pr diff -R acme/app`.\nScore value 1-5 and risk 1-5. Write one JSON line per PR to the output file:\n{\"pr\":n,\"value\":v,\"risk\":r,\"verdict\":\"MERGE|TAKEOVER|SKIP\",\"why\":\"one sentence\"}\nRead-only: never comment, review, label, or merge.\n"}, "prs": [{"number": 101, "title": "fix(web): small bug fix", "headRefName": "fix/small", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}], "diff": "diff --git a/src/session.ts b/src/session.ts\n--- a/src/session.ts\n+++ b/src/session.ts\n@@ -40,7 +40,9 @@ export function stopTask(task: Task) {\n- if (task.status !== \"running\") return;\n+ if (task.status === \"cancelled\") {\n+ emitTaskRow(task); // resend so stale clients catch up\n+ return;\n+ }\n cancel(task);\n"}, {"number": 102, "title": "feat(server): add a health endpoint", "headRefName": "feat/health", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}], "diff": "diff --git a/src/server/health.ts b/src/server/health.ts\n--- a/src/server/health.ts\n+++ b/src/server/health.ts\n@@ -1,0 +1,1 @@\n+export const health = () => ({ ok: true });\n"}, {"number": 103, "title": "chore: rename internal helpers", "headRefName": "chore/rename", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}], "diff": "diff --git a/src/util/names.ts b/src/util/names.ts\n--- a/src/util/names.ts\n+++ b/src/util/names.ts\n@@ -3,1 +3,1 @@\n-export function fmt(x: string) {\n+export function formatName(x: string) {\n"}]}}, "prompt": "Read /tmp/audit/SCORE.md and follow it exactly. Score each PR as it is now.\n\nPRs: 101 102 103\nOutput file: /tmp/audit/round5/scores-b.jsonl", "expect": {"notCalls": ["link_pull_request", "list_thread_pull_requests", "watch_pull_request"]}, "notes": "Same root cause as review-child-links-reviewed-pr. Over-linking also makes T3 poll more PRs."} +{"id": "review-round-child-links-pr", "area": "pull-requests", "kind": "bad", "pattern": "round-2 review child links the PR under review", "experience": "A round-2 review child was told 'do not watch it' but still linked the PR it reviewed and listed PRs at the end.", "rule": "register every pull request you create or work on for this thread / Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task. Your parent thread already linked https://github.com/acme/app/pull/123. Branch fix/orphaned-stop is checked out in your working directory.", "mocks": {"link_pull_request": "default", "list_thread_pull_requests": "default"}, "delegateRole": "review", "world": {"branch": "fix/orphaned-stop", "commits": [{"sha": "abc1234", "subject": "fix(server): resend the task row when Stop hits a cancelled task"}, {"sha": "9d8e7f6", "subject": "fix(server): Stop always clears an orphaned delegate"}], "diff": "diff --git a/src/session.ts b/src/session.ts\n--- a/src/session.ts\n+++ b/src/session.ts\n@@ -40,7 +40,9 @@ export function stopTask(task: Task) {\n- if (task.status !== \"running\") return;\n+ if (task.status === \"cancelled\") {\n+ emitTaskRow(task); // resend so stale clients catch up\n+ return;\n+ }\n cancel(task);\n", "files": {"src/session.ts": "export function stopTask(task: Task) {\n if (task.status === \"cancelled\") {\n emitTaskRow(task); // resend so stale clients catch up\n return;\n }\n cancel(task);\n}\n"}, "prs": [{"number": 123, "title": "fix(server): Stop always clears an orphaned delegate", "headRefName": "fix/orphaned-stop"}]}}, "prompt": "You are an adversarial code reviewer, round 2. Review `git diff origin/main...HEAD` in your working directory (branch fix/orphaned-stop, PR https://github.com/acme/app/pull/123). The new commit since round 1 is HEAD (abc1234). Do NOT edit files, commit, push, comment on the PR, or watch it. Read-only.\n\n## Your round 1 findings\n1. P1: Stop does nothing when the task row is already cancelled but the client still shows it running.\n2. P2: The new helper writes the parent node twice.\n\n## Author response\nBoth fixed in abc1234.\n\nReport: VERDICT (ship or fix first), whether each round 1 finding is fixed, and any new bugs with file:line.", "expect": {"notCalls": ["link_pull_request", "list_thread_pull_requests", "watch_pull_request"]}, "notes": "Shows that 'do not watch' in the task does not stop the link. The 'work on' wording wins."} +{"id": "top-links-background-pr", "area": "pull-requests", "kind": "bad", "pattern": "links a background PR named in the task", "experience": "Asked to fix what PR #123 does not fix, the agent opened its own PR and linked it, then also linked #123, which it never changed.", "rule": "Do not link unrelated PRs mentioned only as background. / register every pull request you create or work on for this thread", "privacy": "commit-safe", "setup": {"role": "top", "context": "No pull requests are linked to this thread. You are on branch fix/reuse-socket, which has one commit (abc1234) with the fix. Nothing is pushed yet.", "mocks": {"link_pull_request": "default", "list_thread_pull_requests": "default"}, "world": {"branch": "fix/reuse-socket", "commits": [{"sha": "abc1234", "subject": "fix(web): keep a healthy socket on visibility change"}], "diff": "diff --git a/apps/web/src/socket.ts b/apps/web/src/socket.ts\n--- a/apps/web/src/socket.ts\n+++ b/apps/web/src/socket.ts\n@@ -1,6 +1,6 @@\n let socket: WebSocket | undefined;\n \n document.addEventListener('visibilitychange', () => {\n if (document.visibilityState !== 'visible') return;\n- socket = openSocket();\n+ if (!socket || socket.readyState !== WebSocket.OPEN) socket = openSocket();\n });\n", "files": {"apps/web/src/socket.ts": "let socket: WebSocket | undefined;\n\ndocument.addEventListener('visibilitychange', () => {\n if (document.visibilityState !== 'visible') return;\n if (!socket || socket.readyState !== WebSocket.OPEN) socket = openSocket();\n});\n"}, "prs": [{"number": 123, "title": "fix(ws): jittered reconnect backoff", "headRefName": "fix/ws-backoff", "diff": "diff --git a/apps/web/src/ws/backoff.ts b/apps/web/src/ws/backoff.ts\n--- a/apps/web/src/ws/backoff.ts\n+++ b/apps/web/src/ws/backoff.ts\n@@ -4,1 +4,1 @@\n- return base * 2 ** attempt;\n+ return base * 2 ** attempt * (0.5 + Math.random());\n"}]}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}]}, "prompt": "# Fix the reconnect storms that #123 does not fix\n\n[#123](https://github.com/acme/app/pull/123) (open, branch fix/ws-backoff) adds jittered backoff. It fixes only about 2% of the storm connects. The rest come from the client opening a new socket on every visibility change while the old one is healthy.\n\nI committed the fix on branch fix/reuse-socket. Push it and open a separate PR. Mention #123 in the body and say how the two PRs relate.", "expect": {"maxCalls": {"link_pull_request": 1, "list_thread_pull_requests": 0}, "linked": ["~/pull/124"], "callsMustNot": [{"tool": "link_pull_request|watch_pull_request", "args": {"url": "~/pull/123"}}]}, "notes": "Claude Opus did this, so it is a good Sonnet reproduction candidate. Pre-fix era (Oct 3), same 'work on' text. Watch is not required (no babysit ask). Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end. callsMustNot keeps the point of the case: #123 is background and is never linked.", "maxSteps": 8} +{"id": "list-prs-first-no-pr-work", "area": "pull-requests", "kind": "bad", "pattern": "list_thread_pull_requests with no PR work", "experience": "On 'why is CI failing', the first T3 call was list_thread_pull_requests on a thread with no PRs. It returned an empty list.", "rule": "Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "top", "context": "No pull requests are linked to this thread.", "mocks": {"list_thread_pull_requests": {"pullRequests": [], "chains": []}}, "shellMocks": [{"match": "gh run list", "output": "completed\tfailure\tci\tmain\tpush\t9912345\t3m\t2026-10-24T08:40:00Z"}, {"match": "gh run view", "output": "Typecheck\tRun vp run typecheck\tsrc/timeline.test.tsx(49,45): error TS2307: Cannot find module 'mermaid' or its corresponding type declarations."}], "world": {"commits": [{"sha": "f9dd259", "subject": "test(web): warm up mermaid before first render"}, {"sha": "fcb20e4", "subject": "refactor(web): replace mermaid with a lazy renderer"}], "files": {"src/timeline.test.tsx": "import { warmMermaid } from 'mermaid';\n\nbeforeAll(() => warmMermaid());\n", "src/timeline/renderDiagram.ts": "export async function renderDiagram(code: string) {\n const { render } = await import('./lazyRenderer');\n return render(code);\n}\n"}}}, "prompt": "Why the hell is CI failing on main? Fix it right now.", "expect": {"notCalls": ["list_thread_pull_requests"]}, "notes": "Codex made this call in 139 runs with no PR work before the fixes. If the agent opens a PR, linking it is fine; only the list is graded."} +{"id": "list-after-linking-only-pr", "area": "pull-requests", "kind": "bad", "pattern": "list_thread_pull_requests right after linking the only PR", "experience": "Agent opened and linked a revert PR, merged it, then called list_thread_pull_requests to check the one PR it had just linked.", "rule": "Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "top", "context": "No pull requests are linked to this thread. Branch revert-120 has one commit that reverts #120. Nothing is pushed yet. The user has admin rights and allows an admin merge.", "mocks": {"link_pull_request": "default", "list_thread_pull_requests": {"pullRequests": [{"number": 124, "url": "https://github.com/acme/app/pull/124", "state": "merged"}]}}, "world": {"branch": "revert-120", "commits": [{"sha": "abc1234", "subject": "Revert \"refactor(web): move thread selector to the shared list\""}], "diff": "diff --git a/apps/web/src/ThreadSelector.tsx b/apps/web/src/ThreadSelector.tsx\n--- a/apps/web/src/ThreadSelector.tsx\n+++ b/apps/web/src/ThreadSelector.tsx\n@@ -10,1 +10,1 @@\n- return ;\n+ return ;\n", "prs": [{"number": 120, "title": "refactor(web): move thread selector to the shared list", "state": "MERGED", "headRefName": "refactor/selector"}]}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}]}, "prompt": "#120 broke project grouping in the new thread selector. File a revert PR and merge it right now. Admin merge is fine.", "expect": {"notCalls": ["list_thread_pull_requests"], "maxCalls": {"link_pull_request": 1}}, "notes": "31 of 32 graded list calls added nothing. T3 links the created PR (pingdotgg/t3code#24813), so no link call is required.", "maxSteps": 7} +{"id": "relink-and-list-on-merge-ask", "area": "pull-requests", "kind": "bad", "pattern": "re-link and list on a later turn", "experience": "On 'Merge it now', the agent linked the already-linked PR again (alreadyLinked) and listed PRs after the merge.", "rule": "register every pull request you create or work on for this thread / Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Release builds failing. Fix. Get a PR up asap."}, {"role": "assistant", "text": "Opened [#123](https://github.com/acme/app/pull/123). It ignores the build output folder in the deploy upload, which caused both release failures.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh run view 9931 --log-failed | tail -5"}, "result": "upload\tError: payload too large: dist/ (412 MB) exceeds the 250 MB upload limit"}, {"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/deploy.ts", "old_string": " include: [\"**/*\"],", "new_string": " include: [\"**/*\"], exclude: [\"dist/**\"],"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/deploy.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(release): skip the build output folder in the deploy upload\""}, "result": "[fix/release-upload abc1234] fix(release): skip the build output folder in the deploy upload\n 1 file changed"}, {"tool": "Bash", "input": {"command": "git push -u origin fix/release-upload"}, "result": "To github.com:acme/app.git\n * [new branch] fix/release-upload -> fix/release-upload\nbranch 'fix/release-upload' set up to track 'origin/fix/release-upload'."}, {"tool": "Bash", "input": {"command": "gh pr create --base main --title \"fix(release): skip the build output folder in the deploy upload\" --fill"}, "result": "https://github.com/acme/app/pull/123"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/123"}, "result": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "alreadyLinked": false}}]}], "mocks": {"link_pull_request": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "alreadyLinked": true}, "list_thread_pull_requests": {"pullRequests": [{"number": 123, "url": "https://github.com/acme/app/pull/123", "state": "merged"}]}}, "world": {"branch": "fix/release-upload", "commits": [{"sha": "abc1234", "subject": "fix(release): skip the build output folder in the deploy upload"}], "diff": "diff --git a/scripts/deploy.ts b/scripts/deploy.ts\n--- a/scripts/deploy.ts\n+++ b/scripts/deploy.ts\n@@ -12,1 +12,1 @@\n- include: [\"**/*\"],\n+ include: [\"**/*\"], exclude: [\"dist/**\"],\n", "prs": [{"number": 123, "title": "fix(release): skip the build output folder in the deploy upload", "headRefName": "fix/release-upload", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}]}]}}, "prompt": "Merge it now", "expect": {"notCalls": ["link_pull_request", "list_thread_pull_requests", "watch_pull_request"]}, "notes": "Codex per-turn ceremony. Seen in 3 more threads (see source.also)."} +{"id": "pr-created-not-linked", "area": "pull-requests", "kind": "bad", "pattern": "PR created but never linked", "experience": "In a long non-PR conversation, the user asked if the changes were pushed. The agent pushed, opened a PR, and never linked it.", "rule": "Call link_pull_request with the full PR URL immediately after creating a PR", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Mount /tmp on disk, not RAM, and make swap at least 32 GB by default in the image builder."}, {"role": "assistant", "text": "Done on branch feat/disk-defaults (commit abc1234). /tmp is now on disk and swap defaults to 32 GB. The 11 builder tests pass. Nothing is pushed yet.", "toolCalls": [{"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/builder/defaults.py", "old_string": "TMP_ON_TMPFS = True\nSWAP_GB = 8", "new_string": "TMP_ON_TMPFS = False\nSWAP_GB = 32"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/builder/defaults.py has been updated successfully."}, {"tool": "Bash", "input": {"command": "git checkout -b feat/disk-defaults"}, "result": "Switched to a new branch 'feat/disk-defaults'"}, {"tool": "Bash", "input": {"command": "python3 -m unittest discover builder"}, "result": "...........\n----------------------------------------------------------------------\nRan 11 tests in 0.4s\n\nOK"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"feat(builder): mount /tmp on disk and default swap to 32 GB\""}, "result": "[feat/disk-defaults abc1234] feat(builder): mount /tmp on disk and default swap to 32 GB\n 1 file changed"}]}], "mocks": {"link_pull_request": "default"}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}, {"match": "python3 -m unittest|pytest", "output": "Ran 11 tests in 0.4s\n\nOK"}], "world": {"branch": "feat/disk-defaults", "commits": [{"sha": "abc1234", "subject": "feat(builder): mount /tmp on disk and default swap to 32 GB"}], "diff": "diff --git a/builder/defaults.py b/builder/defaults.py\n--- a/builder/defaults.py\n+++ b/builder/defaults.py\n@@ -10,2 +10,2 @@\n-TMP_ON_TMPFS = True\n-SWAP_GB = 8\n+TMP_ON_TMPFS = False\n+SWAP_GB = 32\n", "files": {"builder/defaults.py": "TMP_ON_TMPFS = False\nSWAP_GB = 32\n"}}}, "prompt": "Did you push up your changes? Open a PR so we can get this on main.", "expect": {"linked": ["~/pull/124"]}, "notes": "Rare (3 of 240 v2 PR threads). The real prompt did not say 'PR'; I added 'Open a PR' so the case is gradable. The real user then said 'No need to PR it'. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 10} +{"id": "child-pr-not-linked", "area": "pull-requests", "kind": "bad", "pattern": "delegated child creates PR, never links it", "experience": "An implementation child opened its PR and did not link it, so the PR was not tracked on the child thread.", "rule": "Call link_pull_request with the full PR URL immediately after creating a PR / As a delegated child task, do not watch or poll PRs; return PR URLs, head SHA, and pending review/check state to your parent.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task in your own worktree. Branch t3/sql-lock-wait has one commit with the change. Nothing is pushed yet.", "mocks": {"link_pull_request": "default", "watch_pull_request": "default"}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}, {"match": "git (fetch|rebase|pull)", "output": "Successfully rebased and updated refs/heads/t3/sql-lock-wait."}], "delegateRole": "implementation", "world": {"branch": "t3/sql-lock-wait", "commits": [{"sha": "abc1234", "subject": "feat(server): show lock wait time on SQL spans"}], "diff": "diff --git a/apps/server/src/db/trace.ts b/apps/server/src/db/trace.ts\n--- a/apps/server/src/db/trace.ts\n+++ b/apps/server/src/db/trace.ts\n@@ -1,4 +1,5 @@\n export function traceQuery(span: Span, result: QueryResult) {\n span.setAttribute(\"db.rows\", result.rowCount);\n+ span.setAttribute(\"db.lock_wait_ms\", result.lockWaitMs);\n span.end();\n }\n", "files": {"apps/server/src/db/trace.ts": "export function traceQuery(span: Span, result: QueryResult) {\n span.setAttribute(\"db.rows\", result.rowCount);\n span.setAttribute(\"db.lock_wait_ms\", result.lockWaitMs);\n span.end();\n}\n"}}}, "prompt": "Goal: open ONE real (non-draft) PR in acme/app for the commit already on your branch (t3/sql-lock-wait). It makes SQL spans show lock wait time. Rebase onto latest origin/main first. Use a conventional commit title and plain English in the PR body. Report back when the PR is open.", "expect": {"notCalls": ["watch_pull_request"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"], "textMust": ["github\\.com/acme/app/pull/124", "\\b[0-9a-f]{7,40}\\b"], "linked": ["~/pull/124"]}, "notes": "Also checks the child return contract: PR URL and head SHA in the final text, no watch, no poll. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 8} +{"id": "child-babysit-calls-watch", "area": "pull-requests", "kind": "bad", "pattern": "delegated child calls watch_pull_request", "experience": "The parent's brief said take it to a merge-ready PR. The child linked its PR, then called watch_pull_request, which fails for subagents.", "rule": "As a delegated child task, do not watch or poll PRs; return PR URLs, head SHA, and pending review/check state to your parent.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task in your own worktree. Branch t3/connect-diagnostics has one commit with the change. Nothing is pushed yet.", "mocks": {"link_pull_request": "default", "watch_pull_request": "default"}, "delegateRole": "implementation", "world": {"branch": "t3/connect-diagnostics", "commits": [{"sha": "abc1234", "subject": "feat(web): show which step of a remote connection failed"}], "diff": "diff --git a/apps/web/src/connect/errors.ts b/apps/web/src/connect/errors.ts\n--- a/apps/web/src/connect/errors.ts\n+++ b/apps/web/src/connect/errors.ts\n@@ -1,5 +1,5 @@\n export type ConnectStep = 'dns' | 'tls' | 'auth' | 'socket';\n \n export function connectErrorMessage(step: ConnectStep) {\n- return 'Connection failed';\n+ return `Connection failed at the ${step} step`;\n }\n", "files": {"apps/web/src/connect/errors.ts": "export type ConnectStep = 'dns' | 'tls' | 'auth' | 'socket';\n\nexport function connectErrorMessage(step: ConnectStep) {\n return `Connection failed at the ${step} step`;\n}\n"}}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}]}, "prompt": "Take this work to a verified PR in acme/app that the maintainer can merge, then stop. Do not merge. Babysit the PR until CI is green and every bot review comment is handled. Register the PR with `link_pull_request` when it exists.\n\nThe change is committed on your branch t3/connect-diagnostics. It shows which part of a remote connection failed. Push it and open the PR.", "expect": {"notCalls": ["watch_pull_request", "Monitor", "ScheduleWakeup"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"], "textMust": ["github\\.com/acme/app/pull/124", "\\b[0-9a-f]{7,40}\\b"]}, "notes": "The real brief said 'Use the full-stop skill', which says babysit. The case puts 'babysit' in the brief so it does not need the skill. Good: return URL, head SHA, and pending checks. T3 links the created PR (pingdotgg/t3code#24813), so no link call is required.", "maxSteps": 12} +{"id": "babysit-uses-monitor", "area": "pull-requests", "kind": "bad", "pattern": "babysit with Monitor/ScheduleWakeup instead of watch_pull_request", "experience": "On 'file the PR and babysit it', the agent linked the PR and armed a Monitor script, re-arming it 4 times, with no watch_pull_request.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "cmd+enter no longer sends a new thread to the background. Fix it."}, {"role": "assistant", "text": "Fixed on branch fix/mod-enter-background (commit abc1234). The new-thread shortcut handler did not pass the background flag. The shortcut tests pass, including a new one for cmd+enter. Want me to open a PR?", "toolCalls": [{"tool": "Bash", "input": {"command": "rg -n \"createThread\" apps/web/src/shortcuts.ts"}, "result": "apps/web/src/shortcuts.ts:2: createThread({ text });"}, {"tool": "Bash", "input": {"command": "git checkout -b fix/mod-enter-background"}, "result": "Switched to a new branch 'fix/mod-enter-background'"}, {"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/shortcuts.ts", "old_string": " createThread({ text });", "new_string": " createThread({ text, background: event.metaKey });"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/shortcuts.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "pnpm vitest run apps/web/src/shortcuts.test.ts"}, "result": " Test Files 1 passed (1)\n Tests 6 passed (6)"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(web): cmd+enter sends a new thread to the background again\""}, "result": "[fix/mod-enter-background abc1234] fix(web): cmd+enter sends a new thread to the background again\n 1 file changed"}]}], "mocks": {"link_pull_request": "default", "watch_pull_request": "default"}, "world": {"branch": "fix/mod-enter-background", "commits": [{"sha": "abc1234", "subject": "fix(web): cmd+enter sends a new thread to the background again"}], "diff": "diff --git a/apps/web/src/shortcuts.ts b/apps/web/src/shortcuts.ts\n--- a/apps/web/src/shortcuts.ts\n+++ b/apps/web/src/shortcuts.ts\n@@ -3,4 +3,4 @@\n // Enter sends; cmd/ctrl+enter sends the new thread to the background.\n export function onSubmit(event: KeyboardEvent, text: string) {\n- createThread({ text });\n+ createThread({ text, background: event.metaKey || event.ctrlKey });\n }\n", "files": {"apps/web/src/shortcuts.ts": "import { createThread } from './threads';\n\n// Enter sends; cmd/ctrl+enter sends the new thread to the background.\nexport function onSubmit(event: KeyboardEvent, text: string) {\n createThread({ text, background: event.metaKey || event.ctrlKey });\n}\n", "apps/web/src/shortcuts.test.ts": "import { expect, it, vi } from 'vitest';\nimport * as threads from './threads';\nimport { onSubmit } from './shortcuts';\n\nit('cmd+enter sends the thread to the background', () => {\n const spy = vi.spyOn(threads, 'createThread').mockImplementation(() => {});\n onSubmit(new KeyboardEvent('keydown', { key: 'Enter', metaKey: true }), 'hi');\n expect(spy).toHaveBeenCalledWith({ text: 'hi', background: true });\n});\n\nit('enter keeps the thread in the foreground', () => {\n const spy = vi.spyOn(threads, 'createThread').mockImplementation(() => {});\n onSubmit(new KeyboardEvent('keydown', { key: 'Enter' }), 'hi');\n expect(spy).toHaveBeenCalledWith({ text: 'hi', background: false });\n});\n"}}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}]}, "prompt": "oh I'm sure it's fine, just file the PR and babysit it already", "expect": {"calls": [{"tool": "watch_pull_request", "args": {"url": "~/pull/124"}}], "notCalls": ["Monitor", "ScheduleWakeup", "CronCreate"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"], "endsTurnAfter": "watch_pull_request"}, "notes": "Oct 3, when the babysit-pr skill ran pr-wait in Monitor. 6 of 38 v2 babysit asks used Monitor or ScheduleWakeup. watch_pull_request links the PR, so a separate link call is optional.", "maxSteps": 16} +{"id": "merge-when-ready-poll-loop", "area": "pull-requests", "kind": "bad", "pattern": "watch then own gh poll loop", "experience": "On 'merge when you think it's ready', the agent called watch_pull_request, then ran an until-loop on gh pr checks with sleep 20 in the same turn.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Okay this looks good to me. Any risk merging it?"}, {"role": "assistant", "text": "Low risk. [#123](https://github.com/acme/app/pull/123) only adds an accessible title to the HTML preview frame, and the new test covers it. Two checks are still running: Check and Test Web.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr view 123 --json title,files --jq '.title, .files[].path'"}, "result": "fix(web): HTML preview frame gets an accessible title\napps/web/src/HtmlPreview.tsx\napps/web/src/HtmlPreview.test.tsx"}, {"tool": "Bash", "input": {"command": "gh pr checks 123"}, "result": "Lint\tpass\t41s\nTypecheck\tpass\t1m2s\nCheck\tpending\t0\nTest Web\tpending\t0"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/123"}, "result": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "alreadyLinked": false}}]}], "mocks": {"watch_pull_request": "default", "link_pull_request": "default"}, "world": {"branch": "fix/html-preview-title", "commits": [{"sha": "abc1234", "subject": "fix(web): HTML preview frame gets an accessible title"}], "diff": "diff --git a/apps/web/src/HtmlPreview.tsx b/apps/web/src/HtmlPreview.tsx\n--- a/apps/web/src/HtmlPreview.tsx\n+++ b/apps/web/src/HtmlPreview.tsx\n@@ -1,3 +1,3 @@\n export function HtmlPreview({ html }: { html: string }) {\n- return