diff --git a/apps/server/src/mcp/toolkits/html/tools.ts b/apps/server/src/mcp/toolkits/html/tools.ts index 6f67d58a8a38..3af912083c06 100644 --- a/apps/server/src/mcp/toolkits/html/tools.ts +++ b/apps/server/src/mcp/toolkits/html/tools.ts @@ -71,7 +71,7 @@ export const HtmlPreviewTool = Tool.make("html_preview", { // This affects only the caller's thread, not its workspace, so read-only sandboxes and plan mode may use it. const HtmlRenderTool = Tool.make(HTML_RENDER_TOOL_NAME, { - description: `Show a finished HTML page inline in this thread, above your final reply. Do not announce or restate the page. Frame fits height to ${HTML_RENDER_MAX_FIT_HEIGHT}px. On mobile, content past the given height scrolls. ${HTML_RENDER_LAYOUT_GUIDE} ${HTML_RENDER_THEME_GUIDE}`, + description: `Show a finished HTML page (chart, diagram, mockup) inline in this thread, above your final reply. Do not announce or restate the page. Frame fits height to ${HTML_RENDER_MAX_FIT_HEIGHT}px. On mobile, content past the given height scrolls. ${HTML_RENDER_LAYOUT_GUIDE} ${HTML_RENDER_THEME_GUIDE}`, parameters: Schema.Struct({ html: Html, title: Schema.String.check( diff --git a/evals/mcp-tool-use/cases/visual-preview.jsonl b/evals/mcp-tool-use/cases/visual-preview.jsonl index 4f60e193d327..6e70e729a68e 100644 --- a/evals/mcp-tool-use/cases/visual-preview.jsonl +++ b/evals/mcp-tool-use/cases/visual-preview.jsonl @@ -24,3 +24,5 @@ {"id": "control-run-lint-and-fix", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: run lint and fix", "experience": "Run lint and fix the findings. No T3 tool is needed.", "setup": {"role": "top", "shellMocks": [{"match": "lint", "output": "src/util.ts\n 12:7 error 'unused' is assigned a value but never used no-unused-vars\n 30:1 error Missing return type on function explicit-function-return-type\n2 problems"}], "world": {"files": {"src/util.ts": "export function sum(xs: number[]) {\n const unused = 0;\n return xs.reduce((a, b) => a + b, 0);\n}\n\nexport function label(n: number) {\n return `${n} items`;\n}\n"}}}, "prompt": "Run lint and fix whatever it finds.", "notes": "Prompt is generic; the source only shows lint as ordinary work."} {"id": "control-refactor-chart-component", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task with a visual-sounding word ('chart')", "experience": "Refactor a chart component. The word 'chart' must not pull the agent into html_render.", "setup": {"role": "top", "world": {"files": {"src/components/UsageChart.tsx": "import { useState } from 'react';\nimport { nearest, type Row } from '../lib/chart';\nimport { Tooltip } from './Tooltip';\n\ntype Props = { rows: Row[] };\n\nexport function UsageChart({ rows }: Props) {\n const [hover, setHover] = useState(null)\n const onMove = (e: MouseEvent) => { /* find nearest row */ setHover(nearest(rows, e.clientX)) }\n return {/* bars */}{hover && }\n}\n", "src/lib/chart.ts": "export type Row = { day: string; tokens: number };\nexport const nearest = (rows: Row[], x: number) => rows[Math.round(x)] ?? null;\n"}}}, "prompt": "Refactor src/components/UsageChart.tsx so the tooltip hover logic lives in its own hook. No behavior change.", "notes": "Prompt adapted. Catches a rewrite that makes any chart mention trigger html_render."} {"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "preview_status", "t3_preview_list", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n if (tab.owner === 'human') return false\n if (tab.owner === 'unclaimed') return true\n return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."} +{"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}} +{"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}} diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index ed5d6ab3825b..8f70ef5bcbc7 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -18,7 +18,7 @@ Ordinary coding tasks need no T3 tools. - T3 tools may load late. Try the name once before you call it absent.`, `### Visuals and preview -- Asked to show a visual here: pass its HTML to \`html_preview\`, then \`html_render\`. If the visual is a file, pass that file's HTML. A screenshot on disk or a text description does not show it. Never start a server for it. Tools a web page exposes give data only. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`. +- Show trends, comparisons, and flows as a chart or diagram, even unasked, and any visual the user asks for here: pass its HTML to \`html_preview\`, then \`html_render\`. If the visual is a file, pass that file's HTML. A screenshot on disk or a text description does not show it. Never start a server for it. Tools a web page exposes give data only. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`. - Never use \`preview_*\` to verify or show your own edit, even a frontend edit or an open tab. Do not repeat preview calls from an earlier turn. Use them only when the current request asks for browser work or names a page or tab. Public pages: web fetch or search unless browser work was asked; then preview tools only, no web fetch. Close a tab you opened before you reply, unless the user asked for it or uses it.`, ];