From acb6e61caddbb49fa30be065aa492a0ec1535144 Mon Sep 17 00:00:00 2001 From: 0xCUB3 Date: Sun, 18 Oct 2026 06:59:37 -0400 Subject: [PATCH 1/5] fix(server): browser snapshots of hidden tabs are no longer blurry --- apps/server/src/imageMime.test.ts | 13 ++++- apps/server/src/imageMime.ts | 12 +++++ .../src/mcp/toolkits/device/handlers.test.ts | 13 +---- .../src/mcp/toolkits/device/handlers.ts | 13 +---- apps/server/src/preview/ServerBrowser.ts | 49 ++----------------- .../src/preview/ServerBrowserPage.test.ts | 31 +++++++++++- apps/server/src/preview/ServerBrowserPage.ts | 33 +++---------- 7 files changed, 66 insertions(+), 98 deletions(-) diff --git a/apps/server/src/imageMime.test.ts b/apps/server/src/imageMime.test.ts index f9c83400edbcb..ed50b01923ef3 100644 --- a/apps/server/src/imageMime.test.ts +++ b/apps/server/src/imageMime.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vite-plus/test"; -import { inferImageExtension, parseBase64DataUrl } from "./imageMime.ts"; +import { inferImageExtension, parseBase64DataUrl, pngDimensions } from "./imageMime.ts"; describe("imageMime", () => { it("parses base64 data URL with mime type", () => { @@ -86,4 +86,15 @@ describe("imageMime", () => { it("does not read inherited keys from mime extension map", () => { expect(inferImageExtension({ mimeType: "constructor" })).toBe(".bin"); }); + + it("reads PNG dimensions from the IHDR chunk", () => { + const png = new Uint8Array(24); + new DataView(png.buffer).setUint32(0, 0x89504e47); + new DataView(png.buffer).setUint32(4, 0x0d0a1a0a); + new DataView(png.buffer).setUint32(12, 0x49484452); + new DataView(png.buffer).setUint32(16, 1179); + new DataView(png.buffer).setUint32(20, 2556); + expect(pngDimensions(png)).toEqual({ width: 1179, height: 2556 }); + expect(pngDimensions(new Uint8Array([1, 2, 3]))).toEqual({ width: 0, height: 0 }); + }); }); diff --git a/apps/server/src/imageMime.ts b/apps/server/src/imageMime.ts index b180dda51a095..e742048aed1f2 100644 --- a/apps/server/src/imageMime.ts +++ b/apps/server/src/imageMime.ts @@ -134,3 +134,15 @@ export function inferImageExtension(input: { mimeType: string; fileName?: string return ".bin"; } + +export function pngDimensions(png: Uint8Array): { width: number; height: number } { + if (png.length < 24) return { width: 0, height: 0 }; + const view = new DataView(png.buffer, png.byteOffset, png.byteLength); + const isPng = + view.getUint32(0) === 0x89504e47 && + view.getUint32(4) === 0x0d0a1a0a && + view.getUint32(12) === 0x49484452; + return isPng + ? { width: view.getUint32(16), height: view.getUint32(20) } + : { width: 0, height: 0 }; +} diff --git a/apps/server/src/mcp/toolkits/device/handlers.test.ts b/apps/server/src/mcp/toolkits/device/handlers.test.ts index 110d4cbba72a6..603879afa6c7f 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vite-plus/test"; -import { agentDeviceQuickStart, agentDeviceTargetArgs, pngDimensions } from "./handlers.ts"; +import { agentDeviceQuickStart, agentDeviceTargetArgs } from "./handlers.ts"; const device = { hostId: "local", @@ -42,15 +42,4 @@ describe("device tool helpers", () => { expect(text).not.toContain(" agent-device "); expect(text).not.toContain("is on PATH"); }); - - it("reads PNG dimensions from the IHDR chunk", () => { - const png = new Uint8Array(24); - new DataView(png.buffer).setUint32(0, 0x89504e47); - new DataView(png.buffer).setUint32(4, 0x0d0a1a0a); - new DataView(png.buffer).setUint32(12, 0x49484452); - new DataView(png.buffer).setUint32(16, 1179); - new DataView(png.buffer).setUint32(20, 2556); - expect(pngDimensions(png)).toEqual({ width: 1179, height: 2556 }); - expect(pngDimensions(new Uint8Array([1, 2, 3]))).toEqual({ width: 0, height: 0 }); - }); }); diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index d42efd9168c45..7b4738fb31af4 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -11,6 +11,7 @@ import * as Effect from "effect/Effect"; import * as Path from "effect/Path"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ServerConfig from "../../../config.ts"; +import { pngDimensions } from "../../../imageMime.ts"; import { ensureAgentDeviceShim } from "../../../device/AgentDeviceShim.ts"; import { nodeRuntimeUnavailableMessage } from "@t3tools/shared/nodeRuntime"; @@ -245,18 +246,6 @@ const handlers = { ), } satisfies McpToolAccess.Handlers; -export function pngDimensions(png: Uint8Array): { width: number; height: number } { - if (png.length < 24) return { width: 0, height: 0 }; - const view = new DataView(png.buffer, png.byteOffset, png.byteLength); - const isPng = - view.getUint32(0) === 0x89504e47 && - view.getUint32(4) === 0x0d0a1a0a && - view.getUint32(12) === 0x49484452; - return isPng - ? { width: view.getUint32(16), height: view.getUint32(20) } - : { width: 0, height: 0 }; -} - const { device_screenshot, ...standardHandlers } = handlers; export const layerStandard = McpToolAccess.toLayer(DeviceStandardToolkit, standardHandlers); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 5157574f90048..ef90011db71e0 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -247,8 +247,6 @@ interface ViewerState { readonly pressedKeys: Map; readonly pressedButtons: Map<"left" | "middle" | "right", { x: number; y: number }>; readonly push: (output: ServerBrowserViewerOutput) => void; - readonly pause: () => Promise; - readonly resume: () => Promise; scrolledAt: number; requestedSize: { width: number; height: number; order: number } | null; @@ -315,7 +313,6 @@ interface ServerTab { recordingStart: Promise | null; captureLock: Promise; - capturing: number; } interface ServerDownload { @@ -761,7 +758,6 @@ const make = Effect.gen(function* () { navigationGeneration: 0, abortedNavigation: null, captureLock: Promise.resolve(), - capturing: 0, }; if (!desktop) { await page.setViewportSize(fixedViewportSize(tab.setting) ?? UNATTACHED_FILL_VIEWPORT); @@ -1431,10 +1427,9 @@ const make = Effect.gen(function* () { try { await encoder.evaluate(ServerBrowserPage.RECORDING_ENCODER_SCRIPT); // Seed idle pages before returning so an immediate stop has a frame to encode. - const firstFrame = await ServerBrowserPage.captureViewport(tab.page, tab.cdp, { + const firstFrame = await ServerBrowserPage.captureViewport(tab.cdp, { format: "jpeg", quality: RECORDING_SCREENCAST.quality, - scale: 1, }); await encoder.evaluate( ([data, width]) => @@ -1486,29 +1481,6 @@ const make = Effect.gen(function* () { return started; }; - // Scaled captures repaint every screencast; pause them to avoid leaking that frame. - const withScreencastsPaused = (tab: ServerTab, capture: () => Promise): Promise => - withCaptureLock(tab, async () => { - tab.capturing += 1; - const recording = tab.recording; - try { - await Promise.all([ - ...[...tab.viewers].map((viewer) => viewer.pause()), - recording?.session.send("Page.stopScreencast").catch(constVoid), - ]); - return await capture(); - } finally { - tab.capturing -= 1; - - await Promise.all([ - ...[...tab.viewers].map((viewer) => viewer.resume()), - recording && tab.recording === recording - ? recording.session.send("Page.startScreencast", RECORDING_SCREENCAST).catch(constVoid) - : undefined, - ]); - } - }); - const stopRecording = (tab: ServerTab) => withCaptureLock(tab, async () => { const recording = tab.recording; @@ -1894,9 +1866,7 @@ const make = Effect.gen(function* () { return { tabId: tab.tabId, colorScheme }; } case "snapshot": { - return withScreencastsPaused(tab, () => - ServerBrowserPage.snapshot({ ...tab, renderScale: RENDER_SCALE }), - ); + return withCaptureLock(tab, () => ServerBrowserPage.snapshot(tab)); } case "click": { const clickInput = input as PreviewAutomationClickInput; @@ -2196,7 +2166,6 @@ const make = Effect.gen(function* () { const startScreencast = (scale: number) => { screencastScale = scale; screencastParams = screencastParams.then(async () => { - if (tab.capturing > 0) return; await session.send("Page.stopScreencast").catch(constVoid); await session .send("Page.startScreencast", { @@ -2230,13 +2199,6 @@ const make = Effect.gen(function* () { for (const item of dropped.value) if (item._tag === "frame") runFork(item.ack); } }, - pause: () => { - screencastParams = screencastParams.then(() => - session.send("Page.stopScreencast").then(constVoid, constVoid), - ); - return screencastParams; - }, - resume: () => startScreencast(screencastScale), scrolledAt: 0, requestedSize: null, }; @@ -2258,14 +2220,9 @@ const make = Effect.gen(function* () { } broadcastControl(tab); pushFileChooser(tab); - // Full scale: a scaled capture would flash in every other viewer. const pushStill = async () => { const data = await withCaptureLock(tab, () => - ServerBrowserPage.captureViewport(tab.page, session, { - format: "jpeg", - quality, - scale: 1, - }), + ServerBrowserPage.captureViewport(session, { format: "jpeg", quality }), ).catch(() => null); if (data) viewer.push({ _tag: "frame", data: Buffer.from(data, "base64"), ack: Effect.void }); diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 1fbf1bbf15467..4e2378db0e87d 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -35,7 +35,6 @@ describe("server browser element refs", () => { ServerBrowserPage.snapshot({ page, cdp, - renderScale: 1, consoleEntries: [], networkEntries: [], actionTimeline: [], @@ -57,6 +56,36 @@ describe("server browser element refs", () => { }; const repeatedRows = `
    ${Array.from({ length: 5 }, (_, i) => `
  • row ${i + 1}
  • `).join("")}
`; + it("captures snapshots at the page's device scale and reports the PNG's own size", async () => { + // Scaled captures make Chromium re-render at a new scale, which a hidden desktop + // webview can't do, so they came back blurry. Snapshots now keep device pixels. + // Raw CDP captures follow the device scale only when it's forced, as the server does. + const hiDpiBrowser = await chromium.launch({ + headless: true, + args: ["--force-device-scale-factor=2"], + }); + const hiDpi = await hiDpiBrowser.newContext({ + viewport: { width: 1280, height: 800 }, + deviceScaleFactor: 2, + }); + try { + const hiDpiPage = await hiDpi.newPage(); + await hiDpiPage.setContent("

sharp

"); + const { screenshot } = await ServerBrowserPage.snapshot({ + page: hiDpiPage, + cdp: await hiDpi.newCDPSession(hiDpiPage), + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + }); + const png = Buffer.from(screenshot.data, "base64"); + expect([png.readUInt32BE(16), png.readUInt32BE(20)]).toEqual([2560, 1600]); + expect([screenshot.width, screenshot.height]).toEqual([2560, 1600]); + } finally { + await hiDpiBrowser.close(); + } + }); + it("clicks the fifth repeated delete control without touching row one", async () => { await page.setContent(repeatedRows); const result = await takeSnapshot(); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 908de1b6c48be..35a4e396682bc 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -20,11 +20,11 @@ import { import { constVoid } from "effect/Function"; import type { CDPSession, Frame, Locator, Page } from "playwright-core"; import * as NodeCrypto from "node:crypto"; +import { pngDimensions } from "../imageMime.ts"; import { BrowserControlInterrupted } from "./SessionControl.ts"; const MAX_EVALUATION_BYTES = 64_000; const MAX_VISIBLE_TEXT_LENGTH = 20_000; -const MAX_SCREENSHOT_WIDTH = 1280; const WAIT_POLL_MS = 100; export const DIAGNOSTIC_BUFFER_LIMIT = 200; @@ -373,27 +373,15 @@ const maskTree = (tree: string, dotted: RegExp) => }) .join("\n"); -// Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets. +// Device pixels only: a scaled capture makes Chromium re-render at the new scale, which a +// hidden desktop webview can't do, so it returns a blurry upscale of a stale frame. export const captureViewport = async ( - page: Page, cdp: CDPSession, - options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, + options: { readonly format: "png" | "jpeg"; readonly quality?: number }, ) => { - let clip; - if (options.scale < 1) { - const viewport = page.viewportSize() ?? { width: 1280, height: 800 }; - const { cssVisualViewport } = await cdp.send("Page.getLayoutMetrics"); - clip = { - x: cssVisualViewport.pageX, - y: cssVisualViewport.pageY, - ...viewport, - scale: options.scale, - }; - } const { data } = await cdp.send("Page.captureScreenshot", { format: options.format, ...(options.quality === undefined ? {} : { quality: options.quality }), - ...(clip ? { clip } : {}), }); return data; }; @@ -401,7 +389,6 @@ export const captureViewport = async ( type SnapshotInput = { readonly page: Page; readonly cdp: CDPSession; - readonly renderScale: number; readonly consoleEntries: ReadonlyArray; readonly networkEntries: ReadonlyArray; readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"]; @@ -411,8 +398,6 @@ const capture = async ( input: SnapshotInput, deadline: number, ): Promise => { - const viewport = input.page.viewportSize() ?? { width: 1280, height: 800 }; - const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * input.renderScale)); const state = refsFor(input.page); invalidateRefs(input.page); const generation = state.generation; @@ -427,7 +412,7 @@ const capture = async ( "url" | "title" | "loading" | "visibleText" | "interactiveElements" > >, - captureViewport(input.page, input.cdp, { format: "png", scale }), + captureViewport(input.cdp, { format: "png" }), ]); void pageAndScreenshot.catch(constVoid); // Leaves time for the second read; a frame that never loads holds the tree until then. @@ -471,12 +456,8 @@ const capture = async ( consoleEntries: [...input.consoleEntries], networkEntries: [...input.networkEntries], actionTimeline: [...input.actionTimeline], - screenshot: { - mimeType: "image/png", - data, - width: Math.round(viewport.width * input.renderScale * scale), - height: Math.round(viewport.height * input.renderScale * scale), - }, + // Read from the PNG, so the size matches whatever scale the page renders at. + screenshot: { mimeType: "image/png", data, ...pngDimensions(Buffer.from(data, "base64")) }, }; }; From d15d029b6a2352ed7db048c9a074c497cd2e7f79 Mon Sep 17 00:00:00 2001 From: 0xCUB3 Date: Sun, 18 Oct 2026 07:14:13 -0400 Subject: [PATCH 2/5] fix(server): bound browser snapshots after native capture --- apps/server/src/preview/ServerBrowser.ts | 4 ++- .../src/preview/ServerBrowserPage.test.ts | 35 +++++++++++++++---- apps/server/src/preview/ServerBrowserPage.ts | 29 ++++++++++++++- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index ef90011db71e0..6265d56b28f2b 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -1866,7 +1866,9 @@ const make = Effect.gen(function* () { return { tabId: tab.tabId, colorScheme }; } case "snapshot": { - return withCaptureLock(tab, () => ServerBrowserPage.snapshot(tab)); + return withCaptureLock(tab, () => + ServerBrowserPage.snapshot({ ...tab, imagePage: () => contexts.scratchPage() }), + ); } case "click": { const clickInput = input as PreviewAutomationClickInput; diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 4e2378db0e87d..7f70bf7261549 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -5,7 +5,16 @@ import { type CDPSession, type Page, } from "playwright-core"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vite-plus/test"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vite-plus/test"; import { presentAsChrome } from "./ServerBrowserContexts.ts"; import * as ServerBrowserPage from "./ServerBrowserPage.ts"; @@ -35,6 +44,7 @@ describe("server browser element refs", () => { ServerBrowserPage.snapshot({ page, cdp, + imagePage: () => browser.newPage(), consoleEntries: [], networkEntries: [], actionTimeline: [], @@ -56,31 +66,42 @@ describe("server browser element refs", () => { }; const repeatedRows = `
    ${Array.from({ length: 5 }, (_, i) => `
  • row ${i + 1}
  • `).join("")}
`; - it("captures snapshots at the page's device scale and reports the PNG's own size", async () => { + it.each([ + { width: 1280, height: 800, outputHeight: 800 }, + { width: 3840, height: 2160, outputHeight: 720 }, + ])("bounds native snapshots of $width x $height", async ({ width, height, outputHeight }) => { // Scaled captures make Chromium re-render at a new scale, which a hidden desktop - // webview can't do, so they came back blurry. Snapshots now keep device pixels. + // webview can't do, so they came back blurry. Resize only after native capture. // Raw CDP captures follow the device scale only when it's forced, as the server does. const hiDpiBrowser = await chromium.launch({ headless: true, args: ["--force-device-scale-factor=2"], }); const hiDpi = await hiDpiBrowser.newContext({ - viewport: { width: 1280, height: 800 }, + viewport: { width, height }, deviceScaleFactor: 2, }); try { const hiDpiPage = await hiDpi.newPage(); await hiDpiPage.setContent("

sharp

"); + const hiDpiCdp = await hiDpi.newCDPSession(hiDpiPage); + const send = vi.spyOn(hiDpiCdp, "send"); const { screenshot } = await ServerBrowserPage.snapshot({ page: hiDpiPage, - cdp: await hiDpi.newCDPSession(hiDpiPage), + cdp: hiDpiCdp, + imagePage: () => hiDpiBrowser.newPage(), consoleEntries: [], networkEntries: [], actionTimeline: [], }); + expect(send.mock.calls.filter(([method]) => method === "Page.captureScreenshot")).toEqual([ + ["Page.captureScreenshot", { format: "png" }], + ]); + expect(await hiDpiPage.evaluate("devicePixelRatio")).toBe(2); + expect(hiDpiPage.viewportSize()).toEqual({ width, height }); const png = Buffer.from(screenshot.data, "base64"); - expect([png.readUInt32BE(16), png.readUInt32BE(20)]).toEqual([2560, 1600]); - expect([screenshot.width, screenshot.height]).toEqual([2560, 1600]); + expect([png.readUInt32BE(16), png.readUInt32BE(20)]).toEqual([1280, outputHeight]); + expect([screenshot.width, screenshot.height]).toEqual([1280, outputHeight]); } finally { await hiDpiBrowser.close(); } diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 35a4e396682bc..75799f7d03d54 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -25,6 +25,7 @@ import { BrowserControlInterrupted } from "./SessionControl.ts"; const MAX_EVALUATION_BYTES = 64_000; const MAX_VISIBLE_TEXT_LENGTH = 20_000; +const MAX_SCREENSHOT_DIMENSION = 1280; const WAIT_POLL_MS = 100; export const DIAGNOSTIC_BUFFER_LIMIT = 200; @@ -386,9 +387,35 @@ export const captureViewport = async ( return data; }; +// Resize after capture in an isolated encoder page. Scaling CDP's clip changes the +// target's rendering scale; resizing the pixels does not touch the target or its streams. +const captureSnapshot = async (cdp: CDPSession, imagePage: () => Promise) => { + const data = await captureViewport(cdp, { format: "png" }); + const { width, height } = pngDimensions(Buffer.from(data, "base64")); + const scale = Math.min(1, MAX_SCREENSHOT_DIMENSION / Math.max(width, height)); + if (scale === 1) return data; + const encoder = await imagePage(); + try { + return await encoder.evaluate(`(async () => { + const image = new Image(); + image.src = ${JSON.stringify(`data:image/png;base64,${data}`)}; + await image.decode(); + const canvas = document.createElement("canvas"); + canvas.width = ${Math.max(1, Math.round(width * scale))}; + canvas.height = ${Math.max(1, Math.round(height * scale))}; + const context = canvas.getContext("2d"); + context.imageSmoothingQuality = "high"; + context.drawImage(image, 0, 0, canvas.width, canvas.height); + return canvas.toDataURL("image/png").split(",")[1]; + })()`); + } finally { + await encoder.close(); + } +}; type SnapshotInput = { readonly page: Page; readonly cdp: CDPSession; + readonly imagePage: () => Promise; readonly consoleEntries: ReadonlyArray; readonly networkEntries: ReadonlyArray; readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"]; @@ -412,7 +439,7 @@ const capture = async ( "url" | "title" | "loading" | "visibleText" | "interactiveElements" > >, - captureViewport(input.cdp, { format: "png" }), + captureSnapshot(input.cdp, input.imagePage), ]); void pageAndScreenshot.catch(constVoid); // Leaves time for the second read; a frame that never loads holds the tree until then. From f5634c093993dacf043f07a541181ddee35b350d Mon Sep 17 00:00:00 2001 From: 0xCUB3 Date: Sun, 18 Oct 2026 07:22:36 -0400 Subject: [PATCH 3/5] fix(server): resize snapshots without launching another browser --- apps/server/src/preview/ServerBrowser.ts | 4 +- .../src/preview/ServerBrowserPage.test.ts | 15 ++++- apps/server/src/preview/ServerBrowserPage.ts | 62 ++++++++++++------- 3 files changed, 54 insertions(+), 27 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 6265d56b28f2b..ef90011db71e0 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -1866,9 +1866,7 @@ const make = Effect.gen(function* () { return { tabId: tab.tabId, colorScheme }; } case "snapshot": { - return withCaptureLock(tab, () => - ServerBrowserPage.snapshot({ ...tab, imagePage: () => contexts.scratchPage() }), - ); + return withCaptureLock(tab, () => ServerBrowserPage.snapshot(tab)); } case "click": { const clickInput = input as PreviewAutomationClickInput; diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 7f70bf7261549..46d78a1ea289e 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -44,7 +44,6 @@ describe("server browser element refs", () => { ServerBrowserPage.snapshot({ page, cdp, - imagePage: () => browser.newPage(), consoleEntries: [], networkEntries: [], actionTimeline: [], @@ -83,13 +82,20 @@ describe("server browser element refs", () => { }); try { const hiDpiPage = await hiDpi.newPage(); - await hiDpiPage.setContent("

sharp

"); + await hiDpiPage.setContent( + `

sharp

`, + ); + // Hostile page globals and a strict CSP must not interfere with the encoder. + await hiDpiPage.evaluate(`(() => { + globalThis.createImageBitmap = globalThis.OffscreenCanvas = () => { + throw new Error("Do not use the page's encoder globals"); + }; + })()`); const hiDpiCdp = await hiDpi.newCDPSession(hiDpiPage); const send = vi.spyOn(hiDpiCdp, "send"); const { screenshot } = await ServerBrowserPage.snapshot({ page: hiDpiPage, cdp: hiDpiCdp, - imagePage: () => hiDpiBrowser.newPage(), consoleEntries: [], networkEntries: [], actionTimeline: [], @@ -97,6 +103,9 @@ describe("server browser element refs", () => { expect(send.mock.calls.filter(([method]) => method === "Page.captureScreenshot")).toEqual([ ["Page.captureScreenshot", { format: "png" }], ]); + expect(hiDpiBrowser.contexts()).toEqual([hiDpi]); + expect(hiDpi.pages()).toEqual([hiDpiPage]); + expect(await hiDpiPage.locator("body").innerHTML()).toBe("

sharp

"); expect(await hiDpiPage.evaluate("devicePixelRatio")).toBe(2); expect(hiDpiPage.viewportSize()).toEqual({ width, height }); const png = Buffer.from(screenshot.data, "base64"); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 75799f7d03d54..579a5f7feaf2f 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -387,35 +387,55 @@ export const captureViewport = async ( return data; }; -// Resize after capture in an isolated encoder page. Scaling CDP's clip changes the -// target's rendering scale; resizing the pixels does not touch the target or its streams. -const captureSnapshot = async (cdp: CDPSession, imagePage: () => Promise) => { +// Resize pixels in the existing tab's isolated world: no extra browser, DOM changes, +// or rendering-scale override (which would blur hidden desktop webviews). +const captureSnapshot = async (cdp: CDPSession) => { const data = await captureViewport(cdp, { format: "png" }); const { width, height } = pngDimensions(Buffer.from(data, "base64")); const scale = Math.min(1, MAX_SCREENSHOT_DIMENSION / Math.max(width, height)); if (scale === 1) return data; - const encoder = await imagePage(); - try { - return await encoder.evaluate(`(async () => { - const image = new Image(); - image.src = ${JSON.stringify(`data:image/png;base64,${data}`)}; - await image.decode(); - const canvas = document.createElement("canvas"); - canvas.width = ${Math.max(1, Math.round(width * scale))}; - canvas.height = ${Math.max(1, Math.round(height * scale))}; - const context = canvas.getContext("2d"); - context.imageSmoothingQuality = "high"; - context.drawImage(image, 0, 0, canvas.width, canvas.height); - return canvas.toDataURL("image/png").split(",")[1]; - })()`); - } finally { - await encoder.close(); + const { frameTree } = await cdp.send("Page.getFrameTree"); + const { executionContextId } = await cdp.send("Page.createIsolatedWorld", { + frameId: frameTree.frame.id, + worldName: "t3-snapshot-encoder", + }); + const { result, exceptionDetails } = await cdp.send("Runtime.callFunctionOn", { + executionContextId, + functionDeclaration: `async function(data, width, height) { + const bytes = Uint8Array.from(atob(data), c => c.charCodeAt(0)); + const image = await createImageBitmap(new Blob([bytes], { type: "image/png" })); + try { + const canvas = new OffscreenCanvas(width, height); + const context = canvas.getContext("2d"); + context.imageSmoothingQuality = "high"; + context.drawImage(image, 0, 0, width, height); + const blob = await canvas.convertToBlob({ type: "image/png" }); + return await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result.split(",")[1]); + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(blob); + }); + } finally { + image.close(); + } + }`, + arguments: [ + { value: data }, + { value: Math.max(1, Math.round(width * scale)) }, + { value: Math.max(1, Math.round(height * scale)) }, + ], + awaitPromise: true, + returnByValue: true, + }); + if (exceptionDetails || typeof result.value !== "string") { + throw new Error("Snapshot PNG resize failed."); } + return result.value as string; }; type SnapshotInput = { readonly page: Page; readonly cdp: CDPSession; - readonly imagePage: () => Promise; readonly consoleEntries: ReadonlyArray; readonly networkEntries: ReadonlyArray; readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"]; @@ -439,7 +459,7 @@ const capture = async ( "url" | "title" | "loading" | "visibleText" | "interactiveElements" > >, - captureSnapshot(input.cdp, input.imagePage), + captureSnapshot(input.cdp), ]); void pageAndScreenshot.catch(constVoid); // Leaves time for the second read; a frame that never loads holds the tree until then. From 4125293b424da89a45d556c64d6c4df27fcada83 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 03:20:56 +0000 Subject: [PATCH 4/5] fix(server): browser snapshots capture at device pixels and shrink in the tab Tightens the resize to native base64 helpers and a single PNG decode, and keeps one snapshot test that proves the native capture and the 1280px bound. Co-Authored-By: Claude Fable 5.1 --- .../src/preview/ServerBrowserPage.test.ts | 103 +++++++++--------- apps/server/src/preview/ServerBrowserPage.ts | 77 ++++++------- 2 files changed, 92 insertions(+), 88 deletions(-) diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 46d78a1ea289e..227c79c21b03e 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -16,6 +16,7 @@ import { vi, } from "vite-plus/test"; +import { pngDimensions } from "../imageMime.ts"; import { presentAsChrome } from "./ServerBrowserContexts.ts"; import * as ServerBrowserPage from "./ServerBrowserPage.ts"; @@ -65,57 +66,6 @@ describe("server browser element refs", () => { }; const repeatedRows = `
    ${Array.from({ length: 5 }, (_, i) => `
  • row ${i + 1}
  • `).join("")}
`; - it.each([ - { width: 1280, height: 800, outputHeight: 800 }, - { width: 3840, height: 2160, outputHeight: 720 }, - ])("bounds native snapshots of $width x $height", async ({ width, height, outputHeight }) => { - // Scaled captures make Chromium re-render at a new scale, which a hidden desktop - // webview can't do, so they came back blurry. Resize only after native capture. - // Raw CDP captures follow the device scale only when it's forced, as the server does. - const hiDpiBrowser = await chromium.launch({ - headless: true, - args: ["--force-device-scale-factor=2"], - }); - const hiDpi = await hiDpiBrowser.newContext({ - viewport: { width, height }, - deviceScaleFactor: 2, - }); - try { - const hiDpiPage = await hiDpi.newPage(); - await hiDpiPage.setContent( - `

sharp

`, - ); - // Hostile page globals and a strict CSP must not interfere with the encoder. - await hiDpiPage.evaluate(`(() => { - globalThis.createImageBitmap = globalThis.OffscreenCanvas = () => { - throw new Error("Do not use the page's encoder globals"); - }; - })()`); - const hiDpiCdp = await hiDpi.newCDPSession(hiDpiPage); - const send = vi.spyOn(hiDpiCdp, "send"); - const { screenshot } = await ServerBrowserPage.snapshot({ - page: hiDpiPage, - cdp: hiDpiCdp, - consoleEntries: [], - networkEntries: [], - actionTimeline: [], - }); - expect(send.mock.calls.filter(([method]) => method === "Page.captureScreenshot")).toEqual([ - ["Page.captureScreenshot", { format: "png" }], - ]); - expect(hiDpiBrowser.contexts()).toEqual([hiDpi]); - expect(hiDpi.pages()).toEqual([hiDpiPage]); - expect(await hiDpiPage.locator("body").innerHTML()).toBe("

sharp

"); - expect(await hiDpiPage.evaluate("devicePixelRatio")).toBe(2); - expect(hiDpiPage.viewportSize()).toEqual({ width, height }); - const png = Buffer.from(screenshot.data, "base64"); - expect([png.readUInt32BE(16), png.readUInt32BE(20)]).toEqual([1280, outputHeight]); - expect([screenshot.width, screenshot.height]).toEqual([1280, outputHeight]); - } finally { - await hiDpiBrowser.close(); - } - }); - it("clicks the fifth repeated delete control without touching row one", async () => { await page.setContent(repeatedRows); const result = await takeSnapshot(); @@ -428,3 +378,54 @@ describe("server browser drag", () => { } }); }); + +describe("server browser screenshots", () => { + it("captures at native device pixels and downsamples to 1280 px in the same tab", async () => { + // Raw CDP captures follow the device scale only when it's forced, as the server does. + const browser = await chromium.launch({ + headless: true, + args: ["--force-device-scale-factor=2"], + }); + try { + const context = await browser.newContext({ + viewport: { width: 3840, height: 2160 }, + deviceScaleFactor: 2, + }); + const page = await context.newPage(); + await page.setContent( + `

sharp

`, + ); + // The encoder must not depend on anything the page can reach or replace. + await page.evaluate(`globalThis.createImageBitmap = globalThis.OffscreenCanvas = () => { + throw new Error("Do not use the page's encoder globals"); + }`); + const cdp = await context.newCDPSession(page); + const send = vi.spyOn(cdp, "send"); + + const { screenshot } = await ServerBrowserPage.snapshot({ + page, + cdp, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + }); + + // A scaled capture makes Chromium re-render at that scale, which a hidden desktop + // webview can't do, so it returns a blurry upscaled frame. + expect(send.mock.calls.filter(([method]) => method === "Page.captureScreenshot")).toEqual([ + ["Page.captureScreenshot", { format: "png" }], + ]); + const png = Buffer.from(screenshot.data, "base64"); + expect(pngDimensions(png)).toEqual({ width: 1280, height: 720 }); + expect({ width: screenshot.width, height: screenshot.height }).toEqual(pngDimensions(png)); + expect(await page.evaluate("[devicePixelRatio, innerWidth, innerHeight]")).toEqual([ + 2, 3840, 2160, + ]); + expect(await page.locator("body").innerHTML()).toBe("

sharp

"); + expect(browser.contexts()).toEqual([context]); + expect(context.pages()).toEqual([page]); + } finally { + await browser.close(); + } + }); +}); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 579a5f7feaf2f..43891c2bf597d 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -387,52 +387,56 @@ export const captureViewport = async ( return data; }; -// Resize pixels in the existing tab's isolated world: no extra browser, DOM changes, -// or rendering-scale override (which would blur hidden desktop webviews). -const captureSnapshot = async (cdp: CDPSession) => { - const data = await captureViewport(cdp, { format: "png" }); - const { width, height } = pngDimensions(Buffer.from(data, "base64")); - const scale = Math.min(1, MAX_SCREENSHOT_DIMENSION / Math.max(width, height)); - if (scale === 1) return data; +// A page's own encoder globals are not trusted, and its CSP may forbid blob images, so the +// resize runs in an isolated world. Chromium hands back the same world for a name while the +// document lives, so repeated snapshots do not pile up contexts. +const ENCODER_WORLD = "t3-snapshot-encoder"; +const RESIZE_PNG = `async function(data, width, height) { + const source = new Blob([Uint8Array.fromBase64(data)], { type: "image/png" }); + const image = await createImageBitmap(source); + try { + const canvas = new OffscreenCanvas(width, height); + const context = canvas.getContext("2d"); + context.imageSmoothingQuality = "high"; + context.drawImage(image, 0, 0, width, height); + const png = await canvas.convertToBlob({ type: "image/png" }); + return new Uint8Array(await png.arrayBuffer()).toBase64(); + } finally { + image.close(); + } +}`; + +// Captured at device pixels and shrunk afterwards: a forced DPR of 2 on a 4K viewport would +// otherwise return a PNG past 100 MB. +const captureSnapshot = async ( + cdp: CDPSession, +): Promise => { + const native = await captureViewport(cdp, { format: "png" }); + const size = pngDimensions(Buffer.from(native, "base64")); + const scale = Math.min(1, MAX_SCREENSHOT_DIMENSION / Math.max(size.width, size.height)); + if (scale === 1) return { mimeType: "image/png", data: native, ...size }; + const width = Math.max(1, Math.round(size.width * scale)); + const height = Math.max(1, Math.round(size.height * scale)); const { frameTree } = await cdp.send("Page.getFrameTree"); const { executionContextId } = await cdp.send("Page.createIsolatedWorld", { frameId: frameTree.frame.id, - worldName: "t3-snapshot-encoder", + worldName: ENCODER_WORLD, }); const { result, exceptionDetails } = await cdp.send("Runtime.callFunctionOn", { executionContextId, - functionDeclaration: `async function(data, width, height) { - const bytes = Uint8Array.from(atob(data), c => c.charCodeAt(0)); - const image = await createImageBitmap(new Blob([bytes], { type: "image/png" })); - try { - const canvas = new OffscreenCanvas(width, height); - const context = canvas.getContext("2d"); - context.imageSmoothingQuality = "high"; - context.drawImage(image, 0, 0, width, height); - const blob = await canvas.convertToBlob({ type: "image/png" }); - return await new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => resolve(reader.result.split(",")[1]); - reader.onerror = () => reject(reader.error); - reader.readAsDataURL(blob); - }); - } finally { - image.close(); - } - }`, - arguments: [ - { value: data }, - { value: Math.max(1, Math.round(width * scale)) }, - { value: Math.max(1, Math.round(height * scale)) }, - ], + functionDeclaration: RESIZE_PNG, + arguments: [{ value: native }, { value: width }, { value: height }], awaitPromise: true, returnByValue: true, }); if (exceptionDetails || typeof result.value !== "string") { - throw new Error("Snapshot PNG resize failed."); + throw new Error( + `Snapshot resize failed: ${exceptionDetails?.exception?.description ?? exceptionDetails?.text ?? "no image"}`, + ); } - return result.value as string; + return { mimeType: "image/png", data: result.value, width, height }; }; + type SnapshotInput = { readonly page: Page; readonly cdp: CDPSession; @@ -484,7 +488,7 @@ const capture = async ( void iframe.dispose().catch(constVoid); if (frame && unread.includes(frame)) throw tooBusy(); } - const [page, data] = await pageAndScreenshot; + const [page, screenshot] = await pageAndScreenshot; if (state.generation !== generation) throw pageChanged(); const mask = (text: string) => (dotted ? text.replace(dotted, MASKED_VALUE) : text); const accessibilityTree = (dotted ? maskTree(tree, dotted) : tree) @@ -503,8 +507,7 @@ const capture = async ( consoleEntries: [...input.consoleEntries], networkEntries: [...input.networkEntries], actionTimeline: [...input.actionTimeline], - // Read from the PNG, so the size matches whatever scale the page renders at. - screenshot: { mimeType: "image/png", data, ...pngDimensions(Buffer.from(data, "base64")) }, + screenshot, }; }; From 682c377a359d681bf66e3e2aeb039269486f69c8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 03:31:51 +0000 Subject: [PATCH 5/5] fix(server): snapshot encodes the discarded native capture for speed The native PNG only feeds the in-tab resize, so it is encoded for speed, and only its header bytes are decoded to read the size. The test pins the absence of a clip instead of the exact capture call. Co-Authored-By: Claude Fable 5.1 --- apps/server/src/preview/ServerBrowserPage.test.ts | 7 ++++--- apps/server/src/preview/ServerBrowserPage.ts | 13 ++++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 227c79c21b03e..79a2d46eaf773 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -412,9 +412,10 @@ describe("server browser screenshots", () => { // A scaled capture makes Chromium re-render at that scale, which a hidden desktop // webview can't do, so it returns a blurry upscaled frame. - expect(send.mock.calls.filter(([method]) => method === "Page.captureScreenshot")).toEqual([ - ["Page.captureScreenshot", { format: "png" }], - ]); + const captures = send.mock.calls.filter(([method]) => method === "Page.captureScreenshot"); + expect(captures).toHaveLength(1); + expect(captures[0]?.[1]).toMatchObject({ format: "png" }); + expect(captures[0]?.[1]).not.toHaveProperty("clip"); const png = Buffer.from(screenshot.data, "base64"); expect(pngDimensions(png)).toEqual({ width: 1280, height: 720 }); expect({ width: screenshot.width, height: screenshot.height }).toEqual(pngDimensions(png)); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 43891c2bf597d..a0ff05682ec16 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -378,11 +378,16 @@ const maskTree = (tree: string, dotted: RegExp) => // hidden desktop webview can't do, so it returns a blurry upscale of a stale frame. export const captureViewport = async ( cdp: CDPSession, - options: { readonly format: "png" | "jpeg"; readonly quality?: number }, + options: { + readonly format: "png" | "jpeg"; + readonly quality?: number; + readonly optimizeForSpeed?: boolean; + }, ) => { const { data } = await cdp.send("Page.captureScreenshot", { format: options.format, ...(options.quality === undefined ? {} : { quality: options.quality }), + ...(options.optimizeForSpeed ? { optimizeForSpeed: true } : {}), }); return data; }; @@ -411,8 +416,10 @@ const RESIZE_PNG = `async function(data, width, height) { const captureSnapshot = async ( cdp: CDPSession, ): Promise => { - const native = await captureViewport(cdp, { format: "png" }); - const size = pngDimensions(Buffer.from(native, "base64")); + // The native PNG is discarded after the resize, so encode it for speed, not size. + const native = await captureViewport(cdp, { format: "png", optimizeForSpeed: true }); + // The IHDR sits in the first 24 bytes; decoding only those avoids a copy of a large frame. + const size = pngDimensions(Buffer.from(native.slice(0, 32), "base64")); const scale = Math.min(1, MAX_SCREENSHOT_DIMENSION / Math.max(size.width, size.height)); if (scale === 1) return { mimeType: "image/png", data: native, ...size }; const width = Math.max(1, Math.round(size.width * scale));