From f1f8cdf604bb6e892202072b6fe2ce09cf639206 Mon Sep 17 00:00:00 2001 From: Matt Hodgson Date: Thu, 29 Oct 2026 15:35:51 -0400 Subject: [PATCH 1/2] fix(desktop): capture headed browser tabs while macOS is locked --- .../desktop/src/preview/DesktopBrowserHost.ts | 13 ++++++--- apps/desktop/src/preview/GuestCapture.ts | 28 +++++++++++++++++++ apps/web/src/browser/HostedBrowserWebview.tsx | 1 + .../src/browser/hostedBrowserWebviewStyle.ts | 4 ++- 4 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 apps/desktop/src/preview/GuestCapture.ts diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 30f252e3a677e..5735a9d883490 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -5,6 +5,7 @@ import { DesktopBrowserEvent, type DesktopBrowserEvent as DesktopBrowserEventType, } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Context from "effect/Context"; import * as NodePath from "node:path"; import * as Effect from "effect/Effect"; @@ -15,6 +16,7 @@ import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { createCdpRelayConnection, type CdpRelayConnection } from "./CdpRelay.ts"; +import { captureGuestPage } from "./GuestCapture.ts"; import { sendPageCommand } from "./GuestInput.ts"; const encodeEvent = Schema.encodeSync(Schema.fromJsonString(DesktopBrowserEvent)); @@ -104,6 +106,7 @@ export class DesktopBrowserHost extends Context.Service< >()("@t3tools/desktop/preview/DesktopBrowserHost") {} export const make = Effect.gen(function* () { + const hostPlatform = yield* HostProcess.Platform; const outbox = yield* PubSub.unbounded(); const pointers = yield* PubSub.sliding<{ readonly key: DesktopBrowserTabKey; @@ -301,10 +304,12 @@ export const make = Effect.gen(function* () { const relayFor = (tab: AttachedTab) => { if (tab.relay) return tab.relay; const { webContents, debugger: debuggee } = tab.debuggee; - const send = (method: string, params: Record, sessionId?: string) => - sessionId === undefined - ? sendPageCommand(webContents, debuggee, method, params) - : debuggee.sendCommand(method, params, sessionId); + const send = (method: string, params: Record, sessionId?: string) => { + if (sessionId !== undefined) return debuggee.sendCommand(method, params, sessionId); + if (hostPlatform === "darwin" && method === "Page.captureScreenshot") + return captureGuestPage(webContents, debuggee, params); + return sendPageCommand(webContents, debuggee, method, params); + }; const detachedOwners = new Set(); /** Viewers and recordings stream; the agent's own sessions never do. */ const streamingOwners = new Set(); diff --git a/apps/desktop/src/preview/GuestCapture.ts b/apps/desktop/src/preview/GuestCapture.ts new file mode 100644 index 0000000000000..abf274cbfd086 --- /dev/null +++ b/apps/desktop/src/preview/GuestCapture.ts @@ -0,0 +1,28 @@ +/** Native capture supplies unchanged frames when an occluded guest's CDP capture stops. */ +export const captureGuestPage = async ( + webContents: Electron.WebContents, + debuggee: Electron.Debugger, + params: Record, +): Promise => { + const format = params["format"] ?? "png"; + if ( + Object.keys(params).some( + (key) => + !["format", "quality", "optimizeForSpeed", "captureBeyondViewport", "fromSurface"].includes( + key, + ), + ) || + params["clip"] !== undefined || + params["captureBeyondViewport"] === true || + params["fromSurface"] === false || + (format !== "png" && format !== "jpeg") + ) { + return debuggee.sendCommand("Page.captureScreenshot", params); + } + const image = await webContents.capturePage(undefined, { stayHidden: true }); + const data = + format === "jpeg" + ? image.toJPEG(typeof params["quality"] === "number" ? params["quality"] : 80) + : image.toPNG(); + return { data: data.toString("base64") }; +}; diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 70eef380dbff9..832325214b283 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -295,6 +295,7 @@ export function HostedBrowserWebview(props: { const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({ active, renderingActive, + serverDriven: serverDriven && isMacPlatform(navigator.platform), // Electron 43 can permanently blank a macOS webview after `visibility: hidden`. keepPaintableWhenInactive: isMacPlatform(navigator.platform), cornerRadius: presentation.cornerRadius, diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.ts index f37c222b08e9c..71c3234128bee 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.ts @@ -21,6 +21,7 @@ const HIDDEN_BROWSER_WEBVIEW_OFFSET = -100_000; export function resolveHostedBrowserWebviewWrapperStyle(input: { readonly active: boolean; readonly renderingActive: boolean; + readonly serverDriven?: boolean; readonly keepPaintableWhenInactive?: boolean; readonly cornerRadius?: number; readonly zIndex?: number; @@ -34,6 +35,7 @@ export function resolveHostedBrowserWebviewWrapperStyle(input: { keepPaintableWhenInactive = false, rect, renderingActive, + serverDriven = false, zIndex = 30, } = input; if (active && rect) { @@ -48,7 +50,7 @@ export function resolveHostedBrowserWebviewWrapperStyle(input: { }; } - if (renderingActive) { + if (renderingActive || serverDriven) { // Electron stops compositing offscreen guests; keep capture-active guests inside the viewport behind the app. return { left: 0, From d4e4a67b8863a3c2ea18c07544594be32d7eebbc Mon Sep 17 00:00:00 2001 From: Matt Hodgson Date: Thu, 29 Oct 2026 15:35:54 -0400 Subject: [PATCH 2/2] test(desktop): cover macOS native viewport capture and CDP fallback --- .../src/preview/DesktopBrowserHost.test.ts | 46 +++++++++++++++++++ apps/desktop/src/preview/GuestCapture.test.ts | 43 +++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 apps/desktop/src/preview/GuestCapture.test.ts diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 08fefd8d733f5..816047b429e06 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -1,6 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off - Stands in for an Electron debugger. import { describe, expect, it } from "@effect/vitest"; import { DesktopBrowserEvent } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Queue from "effect/Queue"; @@ -73,6 +74,13 @@ const makeDebuggee = ( getURL: () => "http://localhost/", getTitle: () => "Page", getUserAgent: () => "Electron", + capturePage: async () => { + await debuggee.sendCommand("Page.captureScreenshot"); + return { + toPNG: () => Buffer.from("native png"), + toJPEG: () => Buffer.from("native jpeg"), + }; + }, }); return { tab: { @@ -134,6 +142,44 @@ const watchPainting = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) = }); describe("DesktopBrowserHost", () => { + it.effect.each(["darwin", "linux"] as const)( + "uses native viewport capture only on macOS (%s)", + (platform) => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + let cdpCaptures = 0; + let nativeCaptures = 0; + const debuggee = makeDebuggee({ + refuse: (method) => { + if (method !== "Page.captureScreenshot") return false; + cdpCaptures += 1; + return true; + }, + }); + debuggee.tab.webContents.capturePage = async () => { + nativeCaptures += 1; + return { toPNG: () => Buffer.from("native png") } as Electron.NativeImage; + }; + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + yield* host.handleCommandLine( + encodeJson({ + type: "cdp", + ...key, + message: encodeJson({ + id: 1, + method: "Page.captureScreenshot", + params: { format: "png" }, + sessionId: "t3-preview-page", + }), + }), + ); + expect(yield* Queue.take(replies)).toBe(1); + expect(nativeCaptures).toBe(platform === "darwin" ? 1 : 0); + expect(cdpCaptures).toBe(platform === "darwin" ? 0 : 1); + }).pipe(Effect.provideService(HostProcess.Platform, platform), Effect.scoped), + ); + it.effect("announces tabs already attached to a backend that starts later", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/GuestCapture.test.ts b/apps/desktop/src/preview/GuestCapture.test.ts new file mode 100644 index 0000000000000..dc61eaf5a7dd8 --- /dev/null +++ b/apps/desktop/src/preview/GuestCapture.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { captureGuestPage } from "./GuestCapture.ts"; + +const nativePage = () => + ({ + capturePage: async () => ({ + toPNG: () => Buffer.from("native png"), + toJPEG: (quality: number) => Buffer.from(`native jpeg ${quality}`), + }), + }) as unknown as Electron.WebContents; + +const unavailableDebugger = { + sendCommand: async () => { + throw new Error("CDP capture is blocked"); + }, +} as unknown as Electron.Debugger; + +describe("captureGuestPage", () => { + it("returns a viewport frame when CDP capture is unavailable", async () => { + const result = await captureGuestPage(nativePage(), unavailableDebugger, {}); + expect(result).toEqual({ data: Buffer.from("native png").toString("base64") }); + }); + + it("retains JPEG quality for remote viewer still frames", async () => { + const result = await captureGuestPage(nativePage(), unavailableDebugger, { + format: "jpeg", + quality: 65, + }); + expect(result).toEqual({ data: Buffer.from("native jpeg 65").toString("base64") }); + }); + + it("preserves CDP capture options that native viewport capture cannot represent", async () => { + const params = { format: "png", clip: { x: 10, y: 20, width: 30, height: 40, scale: 2 } }; + const debuggee = { + sendCommand: async (method: string, received: unknown) => ({ method, received }), + } as unknown as Electron.Debugger; + expect(await captureGuestPage(nativePage(), debuggee, params)).toEqual({ + method: "Page.captureScreenshot", + received: params, + }); + }); +});