diff --git a/apps/server/src/imageMime.test.ts b/apps/server/src/imageMime.test.ts index f9c83400edbcb..ed50b01923ef3 100644 --- a/apps/server/src/imageMime.test.ts +++ b/apps/server/src/imageMime.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vite-plus/test"; -import { inferImageExtension, parseBase64DataUrl } from "./imageMime.ts"; +import { inferImageExtension, parseBase64DataUrl, pngDimensions } from "./imageMime.ts"; describe("imageMime", () => { it("parses base64 data URL with mime type", () => { @@ -86,4 +86,15 @@ describe("imageMime", () => { it("does not read inherited keys from mime extension map", () => { expect(inferImageExtension({ mimeType: "constructor" })).toBe(".bin"); }); + + it("reads PNG dimensions from the IHDR chunk", () => { + const png = new Uint8Array(24); + new DataView(png.buffer).setUint32(0, 0x89504e47); + new DataView(png.buffer).setUint32(4, 0x0d0a1a0a); + new DataView(png.buffer).setUint32(12, 0x49484452); + new DataView(png.buffer).setUint32(16, 1179); + new DataView(png.buffer).setUint32(20, 2556); + expect(pngDimensions(png)).toEqual({ width: 1179, height: 2556 }); + expect(pngDimensions(new Uint8Array([1, 2, 3]))).toEqual({ width: 0, height: 0 }); + }); }); diff --git a/apps/server/src/imageMime.ts b/apps/server/src/imageMime.ts index b180dda51a095..e742048aed1f2 100644 --- a/apps/server/src/imageMime.ts +++ b/apps/server/src/imageMime.ts @@ -134,3 +134,15 @@ export function inferImageExtension(input: { mimeType: string; fileName?: string return ".bin"; } + +export function pngDimensions(png: Uint8Array): { width: number; height: number } { + if (png.length < 24) return { width: 0, height: 0 }; + const view = new DataView(png.buffer, png.byteOffset, png.byteLength); + const isPng = + view.getUint32(0) === 0x89504e47 && + view.getUint32(4) === 0x0d0a1a0a && + view.getUint32(12) === 0x49484452; + return isPng + ? { width: view.getUint32(16), height: view.getUint32(20) } + : { width: 0, height: 0 }; +} diff --git a/apps/server/src/mcp/toolkits/device/handlers.test.ts b/apps/server/src/mcp/toolkits/device/handlers.test.ts index 110d4cbba72a6..603879afa6c7f 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vite-plus/test"; -import { agentDeviceQuickStart, agentDeviceTargetArgs, pngDimensions } from "./handlers.ts"; +import { agentDeviceQuickStart, agentDeviceTargetArgs } from "./handlers.ts"; const device = { hostId: "local", @@ -42,15 +42,4 @@ describe("device tool helpers", () => { expect(text).not.toContain(" agent-device "); expect(text).not.toContain("is on PATH"); }); - - it("reads PNG dimensions from the IHDR chunk", () => { - const png = new Uint8Array(24); - new DataView(png.buffer).setUint32(0, 0x89504e47); - new DataView(png.buffer).setUint32(4, 0x0d0a1a0a); - new DataView(png.buffer).setUint32(12, 0x49484452); - new DataView(png.buffer).setUint32(16, 1179); - new DataView(png.buffer).setUint32(20, 2556); - expect(pngDimensions(png)).toEqual({ width: 1179, height: 2556 }); - expect(pngDimensions(new Uint8Array([1, 2, 3]))).toEqual({ width: 0, height: 0 }); - }); }); diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index d42efd9168c45..7b4738fb31af4 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -11,6 +11,7 @@ import * as Effect from "effect/Effect"; import * as Path from "effect/Path"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ServerConfig from "../../../config.ts"; +import { pngDimensions } from "../../../imageMime.ts"; import { ensureAgentDeviceShim } from "../../../device/AgentDeviceShim.ts"; import { nodeRuntimeUnavailableMessage } from "@t3tools/shared/nodeRuntime"; @@ -245,18 +246,6 @@ const handlers = { ), } satisfies McpToolAccess.Handlers; -export function pngDimensions(png: Uint8Array): { width: number; height: number } { - if (png.length < 24) return { width: 0, height: 0 }; - const view = new DataView(png.buffer, png.byteOffset, png.byteLength); - const isPng = - view.getUint32(0) === 0x89504e47 && - view.getUint32(4) === 0x0d0a1a0a && - view.getUint32(12) === 0x49484452; - return isPng - ? { width: view.getUint32(16), height: view.getUint32(20) } - : { width: 0, height: 0 }; -} - const { device_screenshot, ...standardHandlers } = handlers; export const layerStandard = McpToolAccess.toLayer(DeviceStandardToolkit, standardHandlers); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 0f134133acc15..2178a6fba2c32 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -248,8 +248,6 @@ interface ViewerState { readonly pressedKeys: Map; readonly pressedButtons: Map<"left" | "middle" | "right", { x: number; y: number }>; readonly push: (output: ServerBrowserViewerOutput) => void; - readonly pause: () => Promise; - readonly resume: () => Promise; scrolledAt: number; requestedSize: { width: number; height: number; order: number } | null; @@ -319,7 +317,6 @@ interface ServerTab { recordingStart: Promise | null; captureLock: Promise; - capturing: number; } interface ServerDownload { @@ -779,7 +776,6 @@ const make = Effect.gen(function* () { navigationGeneration: 0, abortedNavigation: null, captureLock: Promise.resolve(), - capturing: 0, }; if (!desktop) { await page.setViewportSize(fixedViewportSize(tab.setting) ?? UNATTACHED_FILL_VIEWPORT); @@ -1449,10 +1445,9 @@ const make = Effect.gen(function* () { try { await encoder.evaluate(ServerBrowserPage.RECORDING_ENCODER_SCRIPT); // Seed idle pages before returning so an immediate stop has a frame to encode. - const firstFrame = await ServerBrowserPage.captureViewport(tab.page, tab.cdp, { + const firstFrame = await ServerBrowserPage.captureViewport(tab.cdp, { format: "jpeg", quality: RECORDING_SCREENCAST.quality, - scale: 1, }); await encoder.evaluate( ([data, width]) => @@ -1504,29 +1499,6 @@ const make = Effect.gen(function* () { return started; }; - // Scaled captures repaint every screencast; pause them to avoid leaking that frame. - const withScreencastsPaused = (tab: ServerTab, capture: () => Promise): Promise => - withCaptureLock(tab, async () => { - tab.capturing += 1; - const recording = tab.recording; - try { - await Promise.all([ - ...[...tab.viewers].map((viewer) => viewer.pause()), - recording?.session.send("Page.stopScreencast").catch(constVoid), - ]); - return await capture(); - } finally { - tab.capturing -= 1; - - await Promise.all([ - ...[...tab.viewers].map((viewer) => viewer.resume()), - recording && tab.recording === recording - ? recording.session.send("Page.startScreencast", RECORDING_SCREENCAST).catch(constVoid) - : undefined, - ]); - } - }); - const stopRecording = (tab: ServerTab) => withCaptureLock(tab, async () => { const recording = tab.recording; @@ -1930,9 +1902,7 @@ const make = Effect.gen(function* () { return { tabId: tab.tabId, colorScheme }; } case "snapshot": { - return withScreencastsPaused(tab, () => - ServerBrowserPage.snapshot({ ...tab, renderScale: RENDER_SCALE }), - ); + return withCaptureLock(tab, () => ServerBrowserPage.snapshot(tab)); } case "click": { const clickInput = input as PreviewAutomationClickInput; @@ -2232,7 +2202,6 @@ const make = Effect.gen(function* () { const startScreencast = (scale: number) => { screencastScale = scale; screencastParams = screencastParams.then(async () => { - if (tab.capturing > 0) return; await session.send("Page.stopScreencast").catch(constVoid); await session .send("Page.startScreencast", { @@ -2266,13 +2235,6 @@ const make = Effect.gen(function* () { for (const item of dropped.value) if (item._tag === "frame") runFork(item.ack); } }, - pause: () => { - screencastParams = screencastParams.then(() => - session.send("Page.stopScreencast").then(constVoid, constVoid), - ); - return screencastParams; - }, - resume: () => startScreencast(screencastScale), scrolledAt: 0, requestedSize: null, }; @@ -2294,14 +2256,9 @@ const make = Effect.gen(function* () { } broadcastControl(tab); pushFileChooser(tab); - // Full scale: a scaled capture would flash in every other viewer. const pushStill = async () => { const data = await withCaptureLock(tab, () => - ServerBrowserPage.captureViewport(tab.page, session, { - format: "jpeg", - quality, - scale: 1, - }), + ServerBrowserPage.captureViewport(session, { format: "jpeg", quality }), ).catch(() => null); if (data) viewer.push({ _tag: "frame", data: Buffer.from(data, "base64"), ack: Effect.void }); diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 1fbf1bbf15467..79a2d46eaf773 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -5,8 +5,18 @@ import { type CDPSession, type Page, } from "playwright-core"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vite-plus/test"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vite-plus/test"; +import { pngDimensions } from "../imageMime.ts"; import { presentAsChrome } from "./ServerBrowserContexts.ts"; import * as ServerBrowserPage from "./ServerBrowserPage.ts"; @@ -35,7 +45,6 @@ describe("server browser element refs", () => { ServerBrowserPage.snapshot({ page, cdp, - renderScale: 1, consoleEntries: [], networkEntries: [], actionTimeline: [], @@ -369,3 +378,55 @@ describe("server browser drag", () => { } }); }); + +describe("server browser screenshots", () => { + it("captures at native device pixels and downsamples to 1280 px in the same tab", async () => { + // Raw CDP captures follow the device scale only when it's forced, as the server does. + const browser = await chromium.launch({ + headless: true, + args: ["--force-device-scale-factor=2"], + }); + try { + const context = await browser.newContext({ + viewport: { width: 3840, height: 2160 }, + deviceScaleFactor: 2, + }); + const page = await context.newPage(); + await page.setContent( + `

sharp

`, + ); + // The encoder must not depend on anything the page can reach or replace. + await page.evaluate(`globalThis.createImageBitmap = globalThis.OffscreenCanvas = () => { + throw new Error("Do not use the page's encoder globals"); + }`); + const cdp = await context.newCDPSession(page); + const send = vi.spyOn(cdp, "send"); + + const { screenshot } = await ServerBrowserPage.snapshot({ + page, + cdp, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + }); + + // A scaled capture makes Chromium re-render at that scale, which a hidden desktop + // webview can't do, so it returns a blurry upscaled frame. + const captures = send.mock.calls.filter(([method]) => method === "Page.captureScreenshot"); + expect(captures).toHaveLength(1); + expect(captures[0]?.[1]).toMatchObject({ format: "png" }); + expect(captures[0]?.[1]).not.toHaveProperty("clip"); + const png = Buffer.from(screenshot.data, "base64"); + expect(pngDimensions(png)).toEqual({ width: 1280, height: 720 }); + expect({ width: screenshot.width, height: screenshot.height }).toEqual(pngDimensions(png)); + expect(await page.evaluate("[devicePixelRatio, innerWidth, innerHeight]")).toEqual([ + 2, 3840, 2160, + ]); + expect(await page.locator("body").innerHTML()).toBe("

sharp

"); + expect(browser.contexts()).toEqual([context]); + expect(context.pages()).toEqual([page]); + } finally { + await browser.close(); + } + }); +}); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 908de1b6c48be..a0ff05682ec16 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -20,11 +20,12 @@ import { import { constVoid } from "effect/Function"; import type { CDPSession, Frame, Locator, Page } from "playwright-core"; import * as NodeCrypto from "node:crypto"; +import { pngDimensions } from "../imageMime.ts"; import { BrowserControlInterrupted } from "./SessionControl.ts"; const MAX_EVALUATION_BYTES = 64_000; const MAX_VISIBLE_TEXT_LENGTH = 20_000; -const MAX_SCREENSHOT_WIDTH = 1280; +const MAX_SCREENSHOT_DIMENSION = 1280; const WAIT_POLL_MS = 100; export const DIAGNOSTIC_BUFFER_LIMIT = 200; @@ -373,35 +374,79 @@ const maskTree = (tree: string, dotted: RegExp) => }) .join("\n"); -// Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets. +// Device pixels only: a scaled capture makes Chromium re-render at the new scale, which a +// hidden desktop webview can't do, so it returns a blurry upscale of a stale frame. export const captureViewport = async ( - page: Page, cdp: CDPSession, - options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, + options: { + readonly format: "png" | "jpeg"; + readonly quality?: number; + readonly optimizeForSpeed?: boolean; + }, ) => { - let clip; - if (options.scale < 1) { - const viewport = page.viewportSize() ?? { width: 1280, height: 800 }; - const { cssVisualViewport } = await cdp.send("Page.getLayoutMetrics"); - clip = { - x: cssVisualViewport.pageX, - y: cssVisualViewport.pageY, - ...viewport, - scale: options.scale, - }; - } const { data } = await cdp.send("Page.captureScreenshot", { format: options.format, ...(options.quality === undefined ? {} : { quality: options.quality }), - ...(clip ? { clip } : {}), + ...(options.optimizeForSpeed ? { optimizeForSpeed: true } : {}), }); return data; }; +// A page's own encoder globals are not trusted, and its CSP may forbid blob images, so the +// resize runs in an isolated world. Chromium hands back the same world for a name while the +// document lives, so repeated snapshots do not pile up contexts. +const ENCODER_WORLD = "t3-snapshot-encoder"; +const RESIZE_PNG = `async function(data, width, height) { + const source = new Blob([Uint8Array.fromBase64(data)], { type: "image/png" }); + const image = await createImageBitmap(source); + try { + const canvas = new OffscreenCanvas(width, height); + const context = canvas.getContext("2d"); + context.imageSmoothingQuality = "high"; + context.drawImage(image, 0, 0, width, height); + const png = await canvas.convertToBlob({ type: "image/png" }); + return new Uint8Array(await png.arrayBuffer()).toBase64(); + } finally { + image.close(); + } +}`; + +// Captured at device pixels and shrunk afterwards: a forced DPR of 2 on a 4K viewport would +// otherwise return a PNG past 100 MB. +const captureSnapshot = async ( + cdp: CDPSession, +): Promise => { + // The native PNG is discarded after the resize, so encode it for speed, not size. + const native = await captureViewport(cdp, { format: "png", optimizeForSpeed: true }); + // The IHDR sits in the first 24 bytes; decoding only those avoids a copy of a large frame. + const size = pngDimensions(Buffer.from(native.slice(0, 32), "base64")); + const scale = Math.min(1, MAX_SCREENSHOT_DIMENSION / Math.max(size.width, size.height)); + if (scale === 1) return { mimeType: "image/png", data: native, ...size }; + const width = Math.max(1, Math.round(size.width * scale)); + const height = Math.max(1, Math.round(size.height * scale)); + const { frameTree } = await cdp.send("Page.getFrameTree"); + const { executionContextId } = await cdp.send("Page.createIsolatedWorld", { + frameId: frameTree.frame.id, + worldName: ENCODER_WORLD, + }); + const { result, exceptionDetails } = await cdp.send("Runtime.callFunctionOn", { + executionContextId, + functionDeclaration: RESIZE_PNG, + arguments: [{ value: native }, { value: width }, { value: height }], + awaitPromise: true, + returnByValue: true, + }); + if (exceptionDetails || typeof result.value !== "string") { + throw new Error( + `Snapshot resize failed: ${exceptionDetails?.exception?.description ?? exceptionDetails?.text ?? "no image"}`, + ); + } + return { mimeType: "image/png", data: result.value, width, height }; +}; + type SnapshotInput = { readonly page: Page; readonly cdp: CDPSession; - readonly renderScale: number; readonly consoleEntries: ReadonlyArray; readonly networkEntries: ReadonlyArray; readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"]; @@ -411,8 +456,6 @@ const capture = async ( input: SnapshotInput, deadline: number, ): Promise => { - const viewport = input.page.viewportSize() ?? { width: 1280, height: 800 }; - const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * input.renderScale)); const state = refsFor(input.page); invalidateRefs(input.page); const generation = state.generation; @@ -427,7 +470,7 @@ const capture = async ( "url" | "title" | "loading" | "visibleText" | "interactiveElements" > >, - captureViewport(input.page, input.cdp, { format: "png", scale }), + captureSnapshot(input.cdp), ]); void pageAndScreenshot.catch(constVoid); // Leaves time for the second read; a frame that never loads holds the tree until then. @@ -452,7 +495,7 @@ const capture = async ( void iframe.dispose().catch(constVoid); if (frame && unread.includes(frame)) throw tooBusy(); } - const [page, data] = await pageAndScreenshot; + const [page, screenshot] = await pageAndScreenshot; if (state.generation !== generation) throw pageChanged(); const mask = (text: string) => (dotted ? text.replace(dotted, MASKED_VALUE) : text); const accessibilityTree = (dotted ? maskTree(tree, dotted) : tree) @@ -471,12 +514,7 @@ const capture = async ( consoleEntries: [...input.consoleEntries], networkEntries: [...input.networkEntries], actionTimeline: [...input.actionTimeline], - screenshot: { - mimeType: "image/png", - data, - width: Math.round(viewport.width * input.renderScale * scale), - height: Math.round(viewport.height * input.renderScale * scale), - }, + screenshot, }; };