From 4291ee611b7ac3bc9adca57fa133d2e163f29e51 Mon Sep 17 00:00:00 2001 From: Byntham <135178921+Byntham@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:53:08 -0700 Subject: [PATCH 1/2] fix(server): text-only browser snapshots no longer hang on hidden tabs preview_snapshot always captured a screenshot, even when the agent asked for neither includeImage nor save, which is the default. A desktop tab the Browser panel is not showing draws no frames, so Page.captureScreenshot never answers and the whole snapshot failed after 10 seconds, while the text reads it waited on had finished long before. The tool now asks the host for a screenshot only when it will show or save one. Without it, the snapshot skips the capture and the screencast pause around it, and the result has no screenshot field. Refs #16567 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/McpHttpServer.test.ts | 28 +++++++++++++------ apps/server/src/mcp/McpHttpServer.ts | 26 ++++++++++------- .../src/mcp/toolkits/preview/handlers.ts | 10 +++++-- apps/server/src/mcp/toolkits/preview/tools.ts | 2 +- apps/server/src/preview/ServerBrowser.ts | 8 ++++-- .../src/preview/ServerBrowserPage.test.ts | 26 +++++++++++++++++ apps/server/src/preview/ServerBrowserPage.ts | 14 ++++++---- packages/contracts/src/previewAutomation.ts | 15 ++++++---- 8 files changed, 93 insertions(+), 36 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 72d84dff8b9f2..5e2079b6a1e86 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -221,10 +221,15 @@ it.effect.each([{}, { includeImage: false }])( ); it.effect.each([ - { mode: "default", input: {}, images: false }, - { mode: "explicit image", input: { includeImage: true }, images: true }, - { mode: "text only", input: { includeImage: false }, images: false }, -])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) => + { mode: "default", input: {}, images: false, hostInput: { screenshot: false } }, + { mode: "explicit image", input: { includeImage: true }, images: true, hostInput: {} }, + { + mode: "text only", + input: { includeImage: false }, + images: false, + hostInput: { screenshot: false }, + }, +])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images, hostInput }) => Effect.scoped( Effect.gen(function* () { const server = yield* McpServer.McpServer; @@ -255,6 +260,7 @@ it.effect.each([ }; const screenshot = { mimeType: "image/png", width: 1, height: 1 }; let requests = 0; + const hostInputs: Array = []; const events = yield* broker.connect({ clientId: "mcp-image-option-client", environmentId }); yield* Stream.runForEach(events, (event) => { if (event.type === "connected") return Deferred.succeed(connected, undefined); @@ -264,16 +270,19 @@ it.effect.each([ tabId: alternateTabId, threadId, }); - expect(event.request.input).toEqual({}); + hostInputs.push(event.request.input); return broker.respond({ clientId: "mcp-image-option-client", connectionId: event.connectionId, requestId: event.request.requestId, ok: true, + // Like the server browser, the host captures only when it is asked to. result: { ...page, title: `Snapshot ${requests}`, - screenshot: { ...screenshot, data: png }, + ...((event.request.input as { screenshot?: boolean }).screenshot === false + ? {} + : { screenshot: { ...screenshot, data: png } }), }, }); }).pipe(Effect.forkScoped); @@ -289,7 +298,7 @@ it.effect.each([ Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), Effect.provideService(McpSchema.McpServerClient, client), ); - const metadata = { ...page, title: `Snapshot ${call}`, screenshot }; + const metadata = { ...page, title: `Snapshot ${call}`, ...(images ? { screenshot } : {}) }; const { accessibilityTree: _tree, ...boundedMetadata } = metadata; expect(snapshot.isError).toBe(false); expect(snapshot.structuredContent).toEqual({ @@ -328,9 +337,12 @@ it.effect.each([ Effect.provideService(McpSchema.McpServerClient, client), ); expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "text"]); - expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot }); + expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7" }); + expect(nextDefault.structuredContent).not.toHaveProperty("screenshot"); expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree"); expect(requests).toBe(7); + // Only a call that shows or saves the image asks the host to capture one. + expect(hostInputs).toEqual([...Array(6).fill(hostInput), { screenshot: false }]); }), ).pipe(Effect.provide(layerTest)), ); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 5c6c4fece44bb..97a46f1678d3f 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -497,7 +497,7 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot Effect.gen(function* () { const snapshot = encodedResult as SnapshotMetadata & { readonly url: string; - readonly screenshot: { + readonly screenshot?: { readonly mimeType: "image/png"; readonly data: string; readonly width: number; @@ -505,9 +505,11 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot }; }; const { screenshot, ...page } = snapshot; - const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); + const png = screenshot && new Uint8Array(Buffer.from(screenshot.data, "base64")); const screenshotPath = - payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; + png && payload?.save === true + ? yield* saveScreenshot(snapshot.url, png) + : undefined; // Images stay out of tool history unless asked for: providers replay them on every // later request, and some reject inline images outright. const includeImage = payload?.includeImage === true; @@ -524,11 +526,15 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot } const metadata = { ...page, - screenshot: { - mimeType: screenshot.mimeType, - width: screenshot.width, - height: screenshot.height, - }, + ...(screenshot === undefined + ? {} + : { + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, + }), ...(screenshotPath === undefined ? {} : { screenshotPath }), }; const bounded = boundSnapshotMetadata(metadata); @@ -554,8 +560,8 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, }, ]), - ...(includeImage - ? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }] + ...(includeImage && png + ? [{ type: "image" as const, data: png, mimeType: "image/png" }] : []), ], }); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index cf9423ed5a78e..3e0c6807563f7 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -204,8 +204,14 @@ const handlers = { invokeTargeted("setColorScheme", input), ), preview_snapshot: McpToolAccess.readsAsCaller((input) => { - const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {}; - return invokeTargeted("snapshot", operationInput); + const { includeImage, save, ...operationInput } = input ?? {}; + // A hidden desktop tab draws no frames, so a screenshot nobody reads could only stall. + return invokeTargeted( + "snapshot", + includeImage === true || save === true + ? operationInput + : { ...operationInput, screenshot: false }, + ); }), preview_click: McpToolAccess.actsAsCaller((input) => invokeTargeted("click", input, input.timeoutMs), diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 88a260ed6d3cc..665b18b121e09 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -139,7 +139,7 @@ const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = safeBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab, including one the user opened (see preview_status tabs), even while they control it; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Returns page state, semantic elements, diagnostics, action history, and screenshot dimensions. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab, including one the user opened (see preview_status tabs), even while they control it; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Returns page state, semantic elements, diagnostics, and action history, plus screenshot dimensions when includeImage or save is set. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", parameters: Schema.Struct({ ...PreviewAutomationTabTargetInput.fields, includeImage: Schema.optional( diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 5b386e5f0797a..1df70c61ebcb8 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -1968,12 +1968,14 @@ const make = Effect.gen(function* () { return { tabId: tab.tabId, colorScheme }; } case "snapshot": { - return withScreencastsPaused(tab, () => + const screenshot = (input as { readonly screenshot?: boolean }).screenshot !== false; + const take = () => ServerBrowserPage.snapshot({ ...tab, renderScale: tab.desktop ? "display" : RENDER_SCALE, - }), - ); + screenshot, + }); + return screenshot ? withScreencastsPaused(tab, take) : take(); } case "click": { const clickInput = input as PreviewAutomationClickInput; diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 67908bf301e1b..bc2a1b8c0b6d4 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -214,6 +214,32 @@ describe("server browser element refs", () => { expect(tree).not.toContain("PAYLOAD"); }); + it("reads a page that draws no frames when no screenshot is wanted", async () => { + await page.setContent(""); + const methods: Array = []; + // A desktop tab parked offscreen draws no frames, so Chromium never answers its screenshot. + const drawsNothing = { + send: (...[method, params]: Parameters) => { + methods.push(method); + return method === "Page.captureScreenshot" + ? new Promise(() => {}) + : cdp.send(method, params); + }, + } as CDPSession; + const result = await ServerBrowserPage.snapshot({ + page, + cdp: drawsNothing, + renderScale: 1, + screenshot: false, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + }); + buttonLocator(result.accessibilityTree, "continue"); + expect(result.screenshot).toBeUndefined(); + expect(methods).not.toContain("Page.captureScreenshot"); + }); + it("rejects refs from another tab", async () => { await page.setContent(""); const locator = buttonLocator((await takeSnapshot()).accessibilityTree, "same label"); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 02089cca3205c..568db9ca51fec 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -509,6 +509,8 @@ type SnapshotInput = { readonly cdp: CDPSession; /** Headless pages render at the scale the server set; a desktop page at its display's. */ readonly renderScale: number | "display"; + /** A screenshot waits for the page's next frame, which a hidden desktop tab never draws. */ + readonly screenshot?: boolean; readonly consoleEntries: ReadonlyArray; readonly networkEntries: ReadonlyArray; readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"]; @@ -554,7 +556,9 @@ const capture = async ( "url" | "title" | "loading" | "visibleText" | "interactiveElements" > >, - captureViewport(input.page, input.cdp, { format: "png", scale, viewport, offset }), + input.screenshot === false + ? undefined + : captureViewport(input.page, input.cdp, { format: "png", scale, viewport, offset }), ]); void pageAndScreenshot.catch(constVoid); // Leaves time for the second read; a frame that never loads holds the tree until then. @@ -601,11 +605,9 @@ const capture = async ( consoleEntries: [...input.consoleEntries], networkEntries: [...input.networkEntries], actionTimeline: [...input.actionTimeline], - screenshot: { - mimeType: "image/png", - data, - ...pngSize(data), - }, + ...(data === undefined + ? {} + : { screenshot: { mimeType: "image/png", data, ...pngSize(data) } }), }; }; diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 0e9ff94d6ffe5..ca89ca4daba85 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -724,12 +724,15 @@ export const PreviewAutomationSnapshot = Schema.Struct({ consoleEntries: Schema.Array(PreviewAutomationConsoleEntry), networkEntries: Schema.Array(PreviewAutomationNetworkEntry), actionTimeline: Schema.Array(PreviewAutomationActionEvent), - screenshot: Schema.Struct({ - mimeType: Schema.Literal("image/png"), - data: Schema.String, - width: Schema.Int, - height: Schema.Int, - }), + /** Absent when the caller asked for none. */ + screenshot: Schema.optional( + Schema.Struct({ + mimeType: Schema.Literal("image/png"), + data: Schema.String, + width: Schema.Int, + height: Schema.Int, + }), + ), }); export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type; From b7f5eb78d4ea0646577a07be75c962253d531197 Mon Sep 17 00:00:00 2001 From: Byntham <135178921+Byntham@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:53:08 -0700 Subject: [PATCH 2/2] fix(server): describe snapshot dimensions only for image results Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/toolkits/preview/tools.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 665b18b121e09..2538dbbbc9f5d 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -139,7 +139,7 @@ const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = safeBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab, including one the user opened (see preview_status tabs), even while they control it; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Returns page state, semantic elements, diagnostics, and action history, plus screenshot dimensions when includeImage or save is set. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab, including one the user opened (see preview_status tabs), even while they control it; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Returns page state, semantic elements, diagnostics, and action history, plus screenshot dimensions when includeImage is set. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref= to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", parameters: Schema.Struct({ ...PreviewAutomationTabTargetInput.fields, includeImage: Schema.optional(