diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 8b73483380e2c..17d33fe117004 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -220,10 +220,15 @@ it.effect.each([{}, { includeImage: false }])( ); it.effect.each([ - { mode: "default", input: {}, images: false }, - { mode: "explicit image", input: { includeImage: true }, images: true }, - { mode: "text only", input: { includeImage: false }, images: false }, -])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) => + { mode: "default", input: {}, images: false, hostInput: { screenshot: false } }, + { mode: "explicit image", input: { includeImage: true }, images: true, hostInput: {} }, + { + mode: "text only", + input: { includeImage: false }, + images: false, + hostInput: { screenshot: false }, + }, +])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images, hostInput }) => Effect.scoped( Effect.gen(function* () { const server = yield* McpServer.McpServer; @@ -254,6 +259,7 @@ it.effect.each([ }; const screenshot = { mimeType: "image/png", width: 1, height: 1 }; let requests = 0; + const hostInputs: Array = []; const events = yield* broker.connect({ clientId: "mcp-image-option-client", environmentId }); yield* Stream.runForEach(events, (event) => { if (event.type === "connected") return Deferred.succeed(connected, undefined); @@ -263,16 +269,19 @@ it.effect.each([ tabId: alternateTabId, threadId, }); - expect(event.request.input).toEqual({}); + hostInputs.push(event.request.input); return broker.respond({ clientId: "mcp-image-option-client", connectionId: event.connectionId, requestId: event.request.requestId, ok: true, + // Like the server browser, the host captures only when it is asked to. result: { ...page, title: `Snapshot ${requests}`, - screenshot: { ...screenshot, data: png }, + ...((event.request.input as { screenshot?: boolean }).screenshot === false + ? {} + : { screenshot: { ...screenshot, data: png } }), }, }); }).pipe(Effect.forkScoped); @@ -288,7 +297,7 @@ it.effect.each([ Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), Effect.provideService(McpSchema.McpServerClient, client), ); - const metadata = { ...page, title: `Snapshot ${call}`, screenshot }; + const metadata = { ...page, title: `Snapshot ${call}`, ...(images ? { screenshot } : {}) }; const { accessibilityTree: _tree, ...boundedMetadata } = metadata; expect(snapshot.isError).toBe(false); expect(snapshot.structuredContent).toEqual({ @@ -327,9 +336,12 @@ it.effect.each([ Effect.provideService(McpSchema.McpServerClient, client), ); expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "text"]); - expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot }); + expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7" }); + expect(nextDefault.structuredContent).not.toHaveProperty("screenshot"); expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree"); expect(requests).toBe(7); + // Only a call that shows or saves the image asks the host to capture one. + expect(hostInputs).toEqual([...Array(6).fill(hostInput), { screenshot: false }]); }), ).pipe(Effect.provide(layerTest)), ); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 5a4e62e3b2417..ed472bae85708 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -478,7 +478,7 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot Effect.gen(function* () { const snapshot = encodedResult as SnapshotMetadata & { readonly url: string; - readonly screenshot: { + readonly screenshot?: { readonly mimeType: "image/png"; readonly data: string; readonly width: number; @@ -486,9 +486,11 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot }; }; const { screenshot, ...page } = snapshot; - const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); + const png = screenshot && new Uint8Array(Buffer.from(screenshot.data, "base64")); const screenshotPath = - payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; + png && payload?.save === true + ? yield* saveScreenshot(snapshot.url, png) + : undefined; // Providers replay images on later requests, and some reject them; include images only when requested. const includeImage = payload?.includeImage === true; if (screenshotPath !== undefined && !includeImage) { @@ -504,11 +506,15 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot } const metadata = { ...page, - screenshot: { - mimeType: screenshot.mimeType, - width: screenshot.width, - height: screenshot.height, - }, + ...(screenshot === undefined + ? {} + : { + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, + }), ...(screenshotPath === undefined ? {} : { screenshotPath }), }; const bounded = boundSnapshotMetadata(metadata); @@ -534,8 +540,8 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, }, ]), - ...(includeImage - ? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }] + ...(includeImage && png + ? [{ type: "image" as const, data: png, mimeType: "image/png" }] : []), ], }); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 37b4e0470783b..939259d499e86 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -202,8 +202,14 @@ const handlers = { invokeTargeted("setColorScheme", input), ), preview_snapshot: McpToolAccess.readsAsCaller((input) => { - const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {}; - return invokeTargeted("snapshot", operationInput); + const { includeImage, save, ...operationInput } = input ?? {}; + // A capture makes a hidden desktop tab paint until it answers; skip one nobody reads. + return invokeTargeted( + "snapshot", + includeImage === true || save === true + ? operationInput + : { ...operationInput, screenshot: false }, + ); }), preview_click: McpToolAccess.actsAsCaller((input) => invokeTargeted("click", input, input.timeoutMs), diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 4d0602ee928ef..3896c1fef52f8 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -137,7 +137,7 @@ const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = safeBrowserTool( Tool.make("preview_snapshot", { description: - "Use it only when the current request asks for browser work, including checking an edit in the browser. Never use it unasked to verify your own edit, or to repeat a snapshot from an earlier turn. Works on a tab the user is driving. Returns page state, elements, diagnostics, and screenshot size. Target one element with `locator='aria-ref='` from `accessibilityTree`, even in frames. Refs die on navigation, a new snapshot, or handoff. Text caps near 20 KB; on your own or an unclaimed tab, `preview_evaluate` reads more. To show a screenshot: `save=true`, then embed ![alt](screenshotPath).", + "Use it only when the current request asks for browser work, including checking an edit in the browser. Never use it unasked to verify your own edit, or to repeat a snapshot from an earlier turn. Works on a tab the user is driving. Returns page state, elements, and diagnostics, plus screenshot size with `includeImage`. Target one element with `locator='aria-ref='` from `accessibilityTree`, even in frames. Refs die on navigation, a new snapshot, or handoff. Text caps near 20 KB; on your own or an unclaimed tab, `preview_evaluate` reads more. To show a screenshot: `save=true`, then embed ![alt](screenshotPath).", parameters: Schema.Struct({ ...PreviewAutomationTabTargetInput.fields, includeImage: Schema.optional( diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 93f6849bbb995..b8a92065cdab1 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -1958,12 +1958,14 @@ const make = Effect.gen(function* () { return { tabId: tab.tabId, colorScheme }; } case "snapshot": { - return withScreencastsPaused(tab, () => + const screenshot = (input as { readonly screenshot?: boolean }).screenshot !== false; + const take = () => ServerBrowserPage.snapshot({ ...tab, renderScale: tab.desktop ? "display" : RENDER_SCALE, - }), - ); + screenshot, + }); + return screenshot ? withScreencastsPaused(tab, take) : take(); } case "click": { const clickInput = input as PreviewAutomationClickInput; diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index b419d48e36597..80314d7ec6cd8 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -212,6 +212,32 @@ describe("server browser element refs", () => { expect(tree).not.toContain("PAYLOAD"); }); + it("reads a page that draws no frames when no screenshot is wanted", async () => { + await page.setContent(""); + const methods: Array = []; + // Stands in for a hidden tab that never paints, so Chromium never answers its screenshot. + const drawsNothing = { + send: (...[method, params]: Parameters) => { + methods.push(method); + return method === "Page.captureScreenshot" + ? new Promise(() => {}) + : cdp.send(method, params); + }, + } as CDPSession; + const result = await ServerBrowserPage.snapshot({ + page, + cdp: drawsNothing, + renderScale: 1, + screenshot: false, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + }); + buttonLocator(result.accessibilityTree, "continue"); + expect(result.screenshot).toBeUndefined(); + expect(methods).not.toContain("Page.captureScreenshot"); + }); + it("rejects refs from another tab", async () => { await page.setContent(""); const locator = buttonLocator((await takeSnapshot()).accessibilityTree, "same label"); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index d3093bb298a02..0965b31c1d99d 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -492,6 +492,8 @@ type SnapshotInput = { readonly cdp: CDPSession; /** Headless pages render at the scale the server set; a desktop page at its display's. */ readonly renderScale: number | "display"; + /** A screenshot waits for the page's next frame, which a hidden desktop tab must paint. */ + readonly screenshot?: boolean; readonly consoleEntries: ReadonlyArray; readonly networkEntries: ReadonlyArray; readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"]; @@ -536,7 +538,9 @@ const capture = async ( "url" | "title" | "loading" | "visibleText" | "interactiveElements" > >, - captureViewport(input.page, input.cdp, { format: "png", scale, viewport, offset }), + input.screenshot === false + ? undefined + : captureViewport(input.page, input.cdp, { format: "png", scale, viewport, offset }), ]); void pageAndScreenshot.catch(constVoid); // Leaves time for the second read; a frame that never loads holds the tree until then. @@ -581,11 +585,9 @@ const capture = async ( consoleEntries: [...input.consoleEntries], networkEntries: [...input.networkEntries], actionTimeline: [...input.actionTimeline], - screenshot: { - mimeType: "image/png", - data, - ...pngSize(data), - }, + ...(data === undefined + ? {} + : { screenshot: { mimeType: "image/png", data, ...pngSize(data) } }), }; }; diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index abaa03512d1ce..16a4ec675fce7 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -500,12 +500,15 @@ export const PreviewAutomationSnapshot = Schema.Struct({ consoleEntries: Schema.Array(PreviewAutomationConsoleEntry), networkEntries: Schema.Array(PreviewAutomationNetworkEntry), actionTimeline: Schema.Array(PreviewAutomationActionEvent), - screenshot: Schema.Struct({ - mimeType: Schema.Literal("image/png"), - data: Schema.String, - width: Schema.Int, - height: Schema.Int, - }), + /** Absent when the caller asked for none. */ + screenshot: Schema.optional( + Schema.Struct({ + mimeType: Schema.Literal("image/png"), + data: Schema.String, + width: Schema.Int, + height: Schema.Int, + }), + ), }); export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type;