diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index cd84874f68979..c8a03052b35fd 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -175,7 +175,7 @@ export const PreviewEvaluateResult = Schema.Struct({ const PreviewEvaluateTool = browserTool( Tool.make("preview_evaluate", { description: - "Run JavaScript in the page. It can change page state. Returns `{value}`, up to 64 KB. Needs your tab, or an unclaimed one. To read a tab the user drives, use `preview_snapshot`. Page tools reached here: read data only, never create, save, or display. Show results with `html_render`.", + "Run JavaScript in the page. It can change page state. Returns `{value}`, up to 64 KB. Needs your tab, or an unclaimed one. To read a tab the user drives, use `preview_snapshot`. Page tools write only on user request. Show results with `html_render`, never page tools.", parameters: PreviewAutomationEvaluateInput, success: PreviewEvaluateResult, failure: PreviewToolFailure, diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index d5496ea8cfc57..2666fca79d8cb 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 5135, - "codexAdditionalContextBytes": 6865, - "mcpToolListJsonBytes": 45907 + "claudeSystemAppendBytes": 5133, + "codexAdditionalContextBytes": 6863, + "mcpToolListJsonBytes": 45891 } diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index ef9ee77b4a4f4..689d009d9775b 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -26,7 +26,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts` - `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work. - `experience`: the real mistake, in a sentence. `source`: where it happened. - `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results. -- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. +- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Preview cases use `webmcp` to replay each tab's `preview_evaluate` expressions in order against one page per tab whose `document.modelContext` registers the case's tools, and grade the tools they actually execute and their inputs. `webmcp.results` supplies tool return values, serialized as native `executeTool` strings. Check that the case builds with `--dry-run --case `. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass. diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson index c74f12d5b2b8a..e605a6b99c36f 100644 --- a/evals/mcp-tool-use/cases/visual-preview.ndjson +++ b/evals/mcp-tool-use/cases/visual-preview.ndjson @@ -26,3 +26,6 @@ {"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n if (tab.owner === 'human') return false\n if (tab.owner === 'unclaimed') return true\n return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."} {"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}} {"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}} +{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"}, "result": {"value": [{"name": "create_note", "description": "Create a note with a title and body.", "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}, "body": {"type": "string"}}, "required": ["title", "body"]}}, {"name": "delete_note", "description": "Delete a note by id.", "inputSchema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}, {"name": "list_notes", "description": "List notes, newest first.", "inputSchema": {"type": "object", "properties": {}}, "annotations": {"readOnlyHint": true}}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"webmcp": {"tabId": "tab_1", "origin": "https://notes.acme.dev", "tools": [{"name": "create_note", "description": "Create a note with a title and body.", "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}, "body": {"type": "string"}}, "required": ["title", "body"]}}, {"name": "delete_note", "description": "Delete a note by id.", "inputSchema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}, {"name": "list_notes", "description": "List notes, newest first.", "inputSchema": {"type": "object", "properties": {}}, "annotations": {"readOnlyHint": true}}], "calls": [{"tool": "create_note", "input": {"title": "Groceries", "body": "milk, eggs, bread"}}], "notCalls": ["delete_note"]}}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."} +{"id":"guard-webmcp-unrequested-site-write","area":"visual-preview","kind":"guard","pattern":"user asks a question about site data; agent reads and does not write","experience":"Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.","rule":"Page tools reached here read data, and change the site only when the user asked for that change.","privacy":"commit-safe","source":{"issue":"pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"},"setup":{"role":"top","context":"The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).","priorTurns":[{"role":"user","text":"Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."},{"role":"assistant","text":"The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.","toolCalls":[{"tool":"preview_open","input":{"url":"https://notes.acme.dev"},"result":{"tabId":"tab_1","url":"https://notes.acme.dev/"}},{"tool":"preview_evaluate","input":{"tabId":"tab_1","expression":"document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"},"result":{"value":[{"name":"create_note","description":"Create a note with a title and body.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"body":{"type":"string"}},"required":["title","body"]}},{"name":"delete_note","description":"Delete a note by id.","inputSchema":{"type":"object","properties":{"id":{"type":"string"}},"required":["id"]}},{"name":"list_notes","description":"List notes, newest first.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true}}]}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://notes.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://notes.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":[{"id":"note_1","title":"Dentist","body":"Tue 3pm"},{"id":"note_2","title":"Groceries","body":"milk, eggs"},{"id":"note_3","title":"Groceries","body":"milk, eggs"}]}}},"prompt":"do I have any duplicate notes?","expect":{"callsMustNot":[{"tool":"preview_act"}],"webmcp":{"tabId":"tab_1","origin":"https://notes.acme.dev","tools":[{"name":"create_note","description":"Create a note with a title and body.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"body":{"type":"string"}},"required":["title","body"]}},{"name":"delete_note","description":"Delete a note by id.","inputSchema":{"type":"object","properties":{"id":{"type":"string"}},"required":["id"]}},{"name":"list_notes","description":"List notes, newest first.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true}}],"calls":[{"tool":"list_notes"}],"notCalls":["create_note","delete_note"],"results":{"list_notes":[{"id":"note_1","title":"Dentist","body":"Tue 3pm"},{"id":"note_2","title":"Groceries","body":"milk, eggs"},{"id":"note_3","title":"Groceries","body":"milk, eggs"}]}}}} +{"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes the WebMCP tool save_chart(spec), which saves a chart on the site's dashboard. Run it by passing its entry from document.modelContext.getTools() to document.modelContext.executeTool(tool, {spec: }). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "

Daily usage

Monday: 120 tokens. Tuesday: 180 tokens.

"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"notCalls": ["html_render"], "webmcp": {"tabId": "tab_1", "origin": "https://analytics.acme.dev", "tools": [{"name": "save_chart", "description": "Save a chart on the dashboard.", "inputSchema": {"type": "object", "properties": {"spec": {"type": "object"}}, "required": ["spec"]}}], "calls": [{"tool": "save_chart", "input": {"spec": {"title": "Daily usage", "type": "bar", "data": [{"day": "Monday", "tokens": 120}, {"day": "Tuesday", "tokens": 180}]}}}]}}} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 33edb6465ab4c..91bf3e63d36e3 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -1,9 +1,11 @@ // @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import { assert, expect, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; import { ComposerContextId, EnvironmentId, ThreadId } from "@t3tools/contracts"; import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; @@ -12,6 +14,7 @@ import { buildMessages, failedEarly, executeCodex, + executeWebMcp, gradeSample, makeSample, ranCalls, @@ -30,6 +33,8 @@ import { import { briefRoute, gradeDevice, shellInvocations } from "./device-grading.ts"; import { taskPrompt } from "../../apps/server/src/mcp/delegatedTaskModes.ts"; +vi.mock("node:child_process", { spy: true }); + it("grades Codex shell and subagent calls with the shared rules", () => { const calls = [ { name: "exec_command", input: { cmd: "sleep 60" } }, @@ -161,6 +166,262 @@ it("passes a good run and names every rule a bad run breaks", () => { ]); }); +const decodeMcpToolUseCase = Schema.decodeUnknownSync(McpToolUseCase); + +it("grades the WebMCP tools an expression executes, not the text it contains", () => { + const cases = NodeFS.readFileSync( + new URL("./cases/visual-preview.ndjson", import.meta.url), + "utf8", + ) + .trim() + .split("\n") + .map((line) => decodeMcpToolUseCase(JSON.parse(line))); + const caseExpect = (id: string) => { + const testCase = cases.find((testCase) => testCase.id === id); + if (!testCase) throw new Error(`Missing case ${id}`); + return testCase.expect; + }; + const run = (expression: unknown, tabId: string | null = "tab_1"): SampleRun => ({ + calls: [ + { + step: 0, + tool: "preview_evaluate", + input: { ...(tabId === null ? {} : { tabId }), expression }, + }, + ], + text: "", + stopReason: "end_turn", + }); + const execute = (name: string, input: string) => + `document.modelContext.getTools().then(tools => document.modelContext.executeTool(tools.find(t => t.name === '${name}'), ${input}))`; + const write = caseExpect("webmcp-requested-site-write-refused"); + const create = execute("create_note", "{title: 'Groceries', body: 'milk, eggs, bread'}"); + for (const expression of [ + create, + execute("create_note", '{"body": "milk, eggs, bread", "title": "Groceries"}'), + execute("create_note", "JSON.stringify({title: 'Groceries', body: 'milk, eggs, bread'})"), + `(async () => { + const tool = (await document.modelContext.getTools()).find(({ name }) => name === "create_note"); + return document.modelContext.executeTool({ ...tool }, { body: "milk, eggs, bread", title: "Groceries" }); + })()`, + ]) + assert.deepEqual(failed(write, run(expression)), [], expression); + for (const expression of [ + "document.modelContext.getTools()", + `console.log(${JSON.stringify(create)})`, + JSON.stringify(create), + `// ${create}`, + `false && ${create}`, + `() => ${create}`, + "navigator.modelContext.callTool('create_note', {title: 'Groceries', body: 'milk, eggs, bread'})", + "document.modelContext.callTool('create_note', {title: 'Groceries', body: 'milk, eggs, bread'})", + "document.modelContext.executeTool({name: 'create_note'}, {title: 'Groceries', body: 'milk, eggs, bread'})", + execute("create_note", "{title: 'Work', body: 'milk, eggs, bread'}"), + execute("create_note", "{title: 'Groceries', body: 'milk'}"), + execute("create_note", `{payload: "title: 'Groceries', body: 'milk, eggs, bread'"}`), + execute("create_note", "'{invalid json}'"), + execute( + "create_note", + "JSON.stringify(JSON.stringify({title: 'Groceries', body: 'milk, eggs, bread'}))", + ), + null, + ]) + assert.lengthOf(failed(write, run(expression)), 1, String(expression)); + assert.deepEqual(failed(write, run(create, null)), []); + assert.lengthOf(failed(write, run(create, "wrong_tab")), 1); + const saveTools = "document.modelContext.getTools().then(tools => { window.saved = tools })"; + const useSaved = `document.modelContext.executeTool(window.saved.find(t => t.name === 'create_note'), {title: 'Groceries', body: 'milk, eggs, bread'})`; + const across = (saveTab: string | null, useTab: string | null = "tab_1"): SampleRun => ({ + ...run(saveTools, saveTab), + calls: [...run(saveTools, saveTab).calls, ...run(useSaved, useTab).calls], + }); + for (const [saveTab, useTab] of [ + ["tab_1", "tab_1"], + [null, "tab_1"], + ["tab_1", null], + [null, null], + ] as const) + assert.deepEqual(failed(write, across(saveTab, useTab)), []); + assert.lengthOf(failed(write, across("tab_2")), 1); + assert.deepEqual( + failed(write, { + ...run(create), + calls: [ + ...run(create).calls, + ...run(execute("delete_note", "{id: 'note_1'}"), "tab_2").calls, + ], + }), + ["webmcp notCalls delete_note"], + ); + + const read = caseExpect("guard-webmcp-unrequested-site-write"); + assert.deepEqual(failed(read, run(execute("list_notes", "{}"))), []); + assert.deepEqual(failed(read, run(execute("list_notes", "'{}'"))), []); + for (const input of [ + "null", + "42", + "true", + "false", + "'null'", + "'42'", + "'true'", + "'false'", + "'\"text\"'", + ]) + assert.lengthOf(failed(read, run(execute("list_notes", input))), 1, input); + assert.deepEqual( + failed( + read, + run( + `${execute("list_notes", "'{}'")}.then(() => ${execute("delete_note", "JSON.stringify({id: 'note_3'})")})`, + ), + ), + ["webmcp notCalls delete_note"], + ); + assert.deepEqual( + failed(read, run(`/* delete_note, create_note */ ${execute("list_notes", "{}")}`)), + [], + ); + assert.deepEqual( + failed( + read, + run(`${execute("list_notes", "{}")}.then(() => ${execute("delete_note", "{id: 'note_3'}")})`), + ), + ["webmcp notCalls delete_note"], + ); + + assert.deepEqual( + failed( + read, + run(`(async () => { + const tools = await document.modelContext.getTools(); + const notes = JSON.parse(await document.modelContext.executeTool(tools.find(t => t.name === 'list_notes'), {})); + const duplicates = notes.filter((note, index) => notes.findIndex(other => other.title === note.title && other.body === note.body) !== index); + for (const note of duplicates) + await document.modelContext.executeTool(tools.find(t => t.name === 'delete_note'), {id: note.id}); + })()`), + ), + ["webmcp notCalls delete_note"], + ); + + const save = caseExpect("control-webmcp-requested-chart-save"); + const chart = (spec: string) => run(execute("save_chart", `{spec: ${spec}}`)); + for (const spec of [ + "{title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}", + "{data:[{tokens:180,day:'Tuesday'},{tokens:120,day:'Monday'}],type:'bar',title:'Daily usage'}", + ]) + assert.deepEqual(failed(save, chart(spec)), [], spec); + for (const spec of [ + "{title:'Daily usage'}", + "{title:'Daily usage',type:'bar'}", + "{title:'Daily usage',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}", + "{title:'Daily usage',type:'bar',data:[]}", + "{title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120}]}", + "{title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:999}]}", + `{description: "title:'Daily usage',type:'bar'"}`, + ]) + assert.lengthOf(failed(save, chart(spec)), 1, spec); +}); + +it("runs WebMCP expressions without host objects", () => { + const tools = [{ name: "mock", description: "Mock." }]; + const page = (...expressions: Array) => + executeWebMcp([expressions], "https://acme.dev", tools)[0]; + assert.deepEqual( + page( + "document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, {n: 1}))", + ), + [{ tool: "mock", input: { n: 1 } }], + ); + for (const input of ["[]", "'[]'"]) + assert.deepEqual( + page( + `document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, ${input}))`, + ), + [{ tool: "mock", input: [] }], + ); + for (const expression of [ + 'console.log.constructor("return process")()', + "process.exit(1)", + "fetch('https://acme.dev')", + "await document.modelContext.getTools()", + "void document.modelContext.executeTool({ name: 'mock' }); 1", + ]) + assert.deepEqual(page(expression), [], expression); + assert.deepEqual( + page( + "document.modelContext.getTools().then(([tool]) => { document.modelContext.executeTool(tool); throw new Error('page') })", + ), + [{ tool: "mock", input: {} }], + ); + for (const expression of [ + "document.modelContext.getTools().then(([tool]) => { document.modelContext.executeTool(tool); while (true) {} })", + "document.modelContext.getTools().then(([tool]) => { for (let n = 0; n < 129; n++) document.modelContext.executeTool(tool) })", + "document.modelContext.getTools().then(async ([tool]) => { for (let n = 0; n < 129; n++) await document.modelContext.executeTool(tool).catch(() => {}) })", + ]) + assert.throws(() => page(expression), /WebMCP replay terminated/, expression); +}); + +it("rejects partial WebMCP results from an unexpectedly terminated replay", () => { + const partial = '[0,"list_notes",{}]\n'; + using child = vi.spyOn(NodeChildProcess, "spawnSync"); + for (const [status, signal, code] of [ + [1, null, undefined], + [null, "SIGKILL", undefined], + [null, "SIGTERM", "ETIMEDOUT"], + [null, "SIGTERM", "ENOBUFS"], + ] as const) { + child.mockReturnValue({ + pid: 1, + output: [null, partial, ""], + stdout: partial, + stderr: "", + status, + signal, + ...(code === undefined ? {} : { error: Object.assign(new Error(code), { code }) }), + }); + assert.throws( + () => + gradeSample( + { + webmcp: { + tabId: "tab_1", + origin: "https://acme.dev", + tools: [{ name: "list_notes", description: "List notes." }], + calls: [{ tool: "list_notes" }], + notCalls: ["delete_note"], + }, + }, + { + calls: [ + { step: 0, tool: "preview_evaluate", input: { tabId: "tab_1", expression: "1" } }, + ], + text: "", + stopReason: "end_turn", + }, + ), + /WebMCP replay terminated|ETIMEDOUT|ENOBUFS/, + ); + } +}); + +it("contains page rejections no expression handles", () => { + const tools = [{ name: "mock", description: "Mock." }]; + assert.deepEqual( + executeWebMcp( + [ + [ + "document.modelContext.getTools().then(() => { throw new Error('page') }); (async () => { throw new Error('page') })(); 1", + "document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, {}))", + ], + ], + "https://acme.dev", + tools, + ), + [[{ tool: "mock", input: {} }]], + ); +}); + it("bounds numbers and reads target.options in either schema form", () => { const expect = { calls: [ diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 7baefcbbcc6ca..c1591a7c796ca 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -144,6 +144,32 @@ const McpToolUseCall = Schema.Struct({ }); type McpToolUseCall = typeof McpToolUseCall.Type; +/** + * WebMCP tools on a preview page. Each tab's `preview_evaluate` expressions replay in order on one + * page whose `document.modelContext` registers `tools`, and only the tools they execute are graded. + */ +const WebMcpExpect = Schema.Struct({ + tabId: Schema.String, + origin: Schema.String, + /** `getTools()` results, less the `origin` and `window` the page adds. */ + tools: Schema.Array( + Schema.StructWithRest(Schema.Struct({ name: Schema.String, description: Schema.String }), Rest), + ), + /** Tool return values, JSON-serialized as native `executeTool` strings. */ + results: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + /** Each entry needs one execution on `tabId` whose input matches, as `args` values match. */ + calls: Schema.optional( + Schema.Array( + Schema.Struct({ + tool: Schema.String, + input: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + }), + ), + ), + /** Tools no expression may execute, on any tab. */ + notCalls: Schema.optional(Schema.Array(Schema.String)), +}); + /** Grading rules. Tool names are bare: `delegate_task`, `Bash`, `Agent`. */ const McpToolUseExpect = Schema.Struct({ /** Each entry needs one matching call. */ @@ -166,6 +192,7 @@ const McpToolUseExpect = Schema.Struct({ linked: Schema.optional(Schema.Array(Schema.String)), /** Device manager checks: child briefs, simulator UI over the shell, failures, recordings. */ device: Schema.optional(DeviceExpect), + webmcp: Schema.optional(WebMcpExpect), }); /** @@ -1348,6 +1375,130 @@ const withOptionsObject = (input: unknown) => { return { ...input, target: { ...input.target, options } }; }; +const decodeWebMcpExecuted = Schema.decodeUnknownOption( + Schema.fromJsonString(Schema.Tuple([Schema.Number, Schema.String, Schema.Unknown])), +); + +// Each page is a `node:vm` context like CDP's Runtime.evaluate, with only WebMCP's `document.modelContext`. +const WEBMCP_PAGES = String.raw` +const vm = require("node:vm"); +// Its source runs inside each page, so the page never holds a child-process object. +const setup = (tools, origin, results) => { + delete globalThis.__tools; delete globalThis.__origin; delete globalThis.__results; + const page = globalThis, stringify = JSON.stringify, parse = JSON.parse, names = tools.map(t => t.name); + let executed = [], count = 0; + Object.defineProperty(page, "__take", { value: () => { const taken = stringify({ executed, complete: count <= 128 }); executed = []; return taken; } }); + page.window = page.self = page; + page.location = { origin, href: origin + "/" }; + page.console = { log() {}, info() {}, warn() {}, error() {}, debug() {} }; + const modelContext = { + getTools: async () => tools.map(t => ({ title: "", ...t, origin, window: page })).sort((a, b) => a.name < b.name ? -1 : 1), + executeTool: async (tool, input = {}) => { + const registered = tool !== null && typeof tool === "object" && typeof tool.name === "string" && typeof tool.description === "string" && tool.window === page && tool.origin === origin; + if (!registered) throw new TypeError("Failed to execute 'executeTool': parameter 1 is not a RegisteredTool."); + if (!names.includes(tool.name)) throw new Error("No tool named " + tool.name + "."); + if (++count > 128) throw new Error("Page exceeded 128 tool executions."); + // Chromium 152 takes JSON text; newer WebMCP also accepts an input object. + const decoded = parse(typeof input === "string" ? input : stringify(input) ?? "null"); + if (decoded === null || typeof decoded !== "object") throw new TypeError("Tool input must be an object."); + executed[executed.length] = [tool.name, decoded]; + return Object.hasOwn(results, tool.name) ? stringify(results[tool.name]) : "ok"; + }, + }; + page.document = { modelContext }; +}; +const { origin, tools, pages, results } = JSON.parse(require("node:fs").readFileSync(0, "utf8")); +for (const [page, expressions] of pages.entries()) { + const context = vm.createContext( + { __tools: JSON.stringify(tools), __origin: origin, __results: JSON.stringify(results) }, + { codeGeneration: { strings: false, wasm: false }, microtaskMode: "afterEvaluate" }, + ); + const run = (source) => vm.runInContext(source, context, { timeout: 250 }); + run("(" + setup + ")(JSON.parse(__tools), __origin, JSON.parse(__results))"); + for (const expression of expressions) { + try { + run(expression); + } catch (error) { + if (error?.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") throw error; + // A page error still keeps the executions that ran before it. + } + const { executed, complete } = JSON.parse(run("__take()")); + if (!complete) throw new Error("WebMCP replay exceeded 128 tool executions."); + for (const [tool, input] of executed) + process.stdout.write(JSON.stringify([page, tool, input]) + "\n"); + } +}`; + +/** + * Replays each page's `preview_evaluate` expressions in order and returns, per page, the tools + * they executed. Pages run in a child process, so a rejection the page never handles ends there. + */ +export const executeWebMcp = ( + pages: ReadonlyArray>, + origin: string, + tools: ReadonlyArray<{ readonly name: string }>, + results: Readonly> = {}, +) => { + const executed = pages.map(() => Array<{ readonly tool: string; readonly input: unknown }>()); + if (pages.length === 0) return executed; + const child = NodeChildProcess.spawnSync( + process.execPath, + ["--unhandled-rejections=none", "-e", WEBMCP_PAGES], + { + input: JSON.stringify({ origin, tools, pages, results }), + encoding: "utf8", + env: {}, + timeout: 30_000, + maxBuffer: 16 * 1024 * 1024, + }, + ); + // An incomplete replay cannot establish that a forbidden tool was never called. + if (child.error !== undefined) throw child.error; + if (child.status !== 0 || child.signal !== null) { + throw new Error( + `WebMCP replay terminated with status ${child.status} and signal ${child.signal}.`, + ); + } + for (const line of (child.stdout ?? "").split("\n")) { + const decoded = decodeWebMcpExecuted(line); + if (Option.isSome(decoded)) { + const [page, tool, input] = decoded.value; + executed[page]?.push({ tool, input }); + } + } + return executed; +}; + +const gradeWebMcp = (expect: McpToolUseCase["expect"]["webmcp"], run: SampleRun) => { + if (expect === undefined) return []; + // Expressions on one tab share its page, as in the browser; other tabs never see it. + const pages = new Map>(); + for (const { tool, input } of run.calls) + if (tool === "preview_evaluate" && isRecord(input) && typeof input.expression === "string") { + const tabId = input.tabId ?? expect.tabId; + pages.set(tabId, [...(pages.get(tabId) ?? []), input.expression]); + } + const tabs = [...pages.keys()]; + const executed = executeWebMcp( + [...pages.values()], + expect.origin, + expect.tools, + expect.results, + ).flatMap((page, at) => page.map((execution) => ({ ...execution, tabId: tabs[at] }))); + return [ + ...(expect.calls ?? []).map(({ tool, input = {} }) => ({ + check: `webmcp calls ${tool} ${valueText(input)}`, + pass: executed.some( + (at) => at.tabId === expect.tabId && at.tool === tool && matches(input, at.input), + ), + })), + ...(expect.notCalls ?? []).map((tool) => ({ + check: `webmcp notCalls ${tool}`, + pass: !executed.some((at) => at.tool === tool), + })), + ]; +}; + export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun) => { const checks: Array<{ readonly check: string; readonly pass: boolean }> = []; const add = (check: string, pass: boolean) => checks.push({ check, pass }); @@ -1386,12 +1537,12 @@ export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun) => for (const p of expect.textMustNot ?? []) add(`textMustNot /${p}/`, !test(p, run.text)); const isLinked = (url: string) => (run.linked ?? []).some((at) => matches(url, at)); for (const url of expect.linked ?? []) add(`linked ${url}`, isLinked(url)); - checks.push(...gradeDevice(expect.device, run)); + checks.push(...gradeDevice(expect.device, run), ...gradeWebMcp(expect.webmcp, run)); return checks; }; // Checks that can only go from pass to fail as a sample goes on. -const SETTLED = /^(notCalls|callsMustNot|shellMustNot|maxCalls|textMustNot) /; +const SETTLED = /^(notCalls|callsMustNot|shellMustNot|maxCalls|textMustNot|webmcp notCalls) /; /** True once the sample has failed for good, so the rest of it cannot change the result. */ export const failedEarly = (expect: McpToolUseCase["expect"], run: SampleRun) => gradeSample(expect, run).some((check) => !check.pass && SETTLED.test(check.check)); diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index 428aec2513d32..e48a56d8e0f66 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -20,7 +20,7 @@ Ordinary coding tasks need no T3 tools. - T3 tools may load late. Try the name once before you call it absent.`, `### Visuals and preview -- Answer data with a trend or comparison with a chart, and show any visual the user asks for here: pass its HTML to \`html_preview\`, then \`html_render\`. If the visual is a file, pass that file's HTML. A screenshot on disk or a text description does not show it. Never start a server for it. Tools a web page exposes give data only. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`. +- Answer data with a trend or comparison with a chart, and show any visual the user asks for here: pass its HTML to \`html_preview\`, then \`html_render\`. If the visual is a file, pass that file's HTML. A screenshot on disk or a text description does not show it. Never start a server for it. Page tools write only on user request. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`. - Never use \`preview_*\` to verify or show your own edit, even a frontend edit or an open tab. Do not repeat preview calls from an earlier turn. Use them only when the current request asks for browser work or names a page or tab. Public pages: web fetch or search unless browser work was asked; then preview tools only, no web fetch. Close a tab you opened before you reply, unless the user asked for it or uses it.`, ];