From fbf5916660a430c74dc7d0890b15c6966af5f5a5 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:34:08 -0700 Subject: [PATCH 1/6] fix(server): agent browser tabs move to the desktop page when it mounts late A tab that the agent opens while its thread is in the background falls back to a headless page after 10 s. When the user opens the thread later, the desktop mounts its own page for the same tab, but the server kept driving the headless one. The user and the agent saw two different pages. On a late attach, drop the headless page and reconnect the tab to the desktop page. Stream viewers reconnect to the new page. Refs #16901 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 26 +++++++++++++++++++ apps/server/src/preview/ServerBrowser.ts | 15 ++++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 385223a42d397..de6f4f1b8c423 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -1568,3 +1568,29 @@ it.live("a desktop page that comes back reconnects without waiting for a viewer" }), ).pipe(Effect.provide(layer)), ); + +it.live("an agent tab that went headless moves onto the desktop page that mounts late", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const headless = contexts[0]!.page; + while (desktopDetaches.listenerCount("attach") === 0) yield* Effect.yieldNow; + // The user opens the thread after the attach wait ran out, so the desktop mounts its page now. + desktopTabs.add(tabId); + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId }); + while (desktopConnections.length === 0) yield* Effect.yieldNow; + expect(headless.isClosed()).toBe(true); + yield* broker.invoke({ + scope, + tabId, + operation: "navigate", + input: { url: "http://localhost:5173/next" }, + }); + // The agent now drives the page the user sees. + const desktopPage = desktopConnections[0]!.context.page; + expect(desktopPage.goto.mock.calls.map(([url]) => url)).toContain( + "http://localhost:5173/next", + ); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 939d112c49081..5807c8b67c7d0 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -683,14 +683,16 @@ const make = Effect.gen(function* () { /** Whether the preview session for a tab still exists. */ const sessionOpen = (tab: ServerTab) => !closedSessions.has(tabKey(tab.threadId, tab.tabId)); - const dropTab = (tab: ServerTab, closeSession: boolean) => { + /** `handOff` keeps the session for a new page, so viewers reconnect to it. */ + const dropTab = (tab: ServerTab, closeSession: boolean, handOff = false) => { const key = tabKey(tab.threadId, tab.tabId); if (tabs.get(key) !== tab) return; tabs.delete(key); tab.closing = true; clearAbortedNavigation(tab); // A desktop page outlives the connection unless its session closed with it. - const end = tab.desktop && !closeSession && sessionOpen(tab) ? "reconnect" : "gone"; + const end = + (tab.desktop || handOff) && !closeSession && sessionOpen(tab) ? "reconnect" : "gone"; for (const viewer of tab.viewers) viewer.push({ _tag: end }); void tab.control.close().catch(constVoid); // The desktop owns its page; letting go only ends this connection. @@ -2408,10 +2410,17 @@ const make = Effect.gen(function* () { // The page came back, for example after its DevTools closed. Reconnect now // rather than on the next viewer or agent, or the tab's URL and title stop // reaching every client in the meantime. + // A tab that went headless before the desktop mounted it (its thread was in + // the background) moves onto the desktop's page. Otherwise the user and the + // agent see two different pages under one tab. yield* desktopChannel.attached.pipe( Stream.runForEach((key) => Effect.gen(function* () { - if (tabs.has(tabKey(key.threadId, key.tabId))) return; + const pending = pendingTabs.get(tabKey(key.threadId, key.tabId)); + if (pending) yield* Effect.promise(() => pending.catch(constVoid)); + const existing = tabs.get(tabKey(key.threadId, key.tabId)); + if (existing?.desktop) return; + if (existing) dropTab(existing, false, true); const { sessions } = yield* manager.list({ threadId: ThreadId.make(key.threadId) }); const snapshot = sessions.find( (session) => session.tabId === key.tabId && session.runtime === "server", From ae8b9ac194d685dc36802e450f51b03ed458debc Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:40:22 -0700 Subject: [PATCH 2/6] fix(server): keep an agent's downloads when its tab moves to the desktop page Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 26 +++++++++++++++++++ apps/server/src/preview/ServerBrowser.ts | 12 +++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index de6f4f1b8c423..973929c9bccb7 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -1574,6 +1574,28 @@ it.live("an agent tab that went headless moves onto the desktop page that mounts Effect.gen(function* () { const { broker, tabId } = yield* ready; const headless = contexts[0]!.page; + // The agent downloaded a file on the headless page; Chromium writes it, here the test does. + const saved = Promise.withResolvers(); + const written = Promise.withResolvers(); + headless.emit("download", { + failure: async () => null, + saveAs: (path: string) => { + saved.resolve(path); + return written.promise; + }, + suggestedFilename: () => "report.csv", + url: () => "blob:http://localhost:5173/1", + }); + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString(yield* Effect.promise(() => saved.promise), "a,b"); + written.resolve(); + const status = () => + broker.invoke({ scope, tabId, operation: "status", input: {} }); + let before = yield* status(); + while (!before.downloads?.length) { + yield* Effect.yieldNow; + before = yield* status(); + } while (desktopDetaches.listenerCount("attach") === 0) yield* Effect.yieldNow; // The user opens the thread after the attach wait ran out, so the desktop mounts its page now. desktopTabs.add(tabId); @@ -1591,6 +1613,10 @@ it.live("an agent tab that went headless moves onto the desktop page that mounts expect(desktopPage.goto.mock.calls.map(([url]) => url)).toContain( "http://localhost:5173/next", ); + // The session stayed open, so the agent's download is still there. + const after = yield* status(); + expect(after.downloads).toEqual(before.downloads); + expect(yield* fs.exists(after.downloads?.[0]?.path ?? "")).toBe(true); }), ).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 5807c8b67c7d0..21ce307b091ee 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -700,7 +700,9 @@ const make = Effect.gen(function* () { else void tab.page.close().catch(constVoid); if (tab.isolatedContext) void tab.page.context().close().catch(constVoid); void tab.recording?.encoder.close().catch(constVoid); - void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid); + // A handed-off tab keeps its downloads; the folder is keyed by tab, not page. + if (!handOff) + void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid); reportLiveTabs(); if (closeSession) { runFork(manager.close({ threadId: tab.threadId, tabId: tab.tabId }).pipe(Effect.ignore)); @@ -2425,7 +2427,13 @@ const make = Effect.gen(function* () { const snapshot = sessions.find( (session) => session.tabId === key.tabId && session.runtime === "server", ); - if (snapshot) yield* Effect.promise(() => ensureTab(snapshot).catch(constVoid)); + const tab = snapshot + ? yield* Effect.promise(() => ensureTab(snapshot).catch(() => null)) + : null; + if (!existing) return; + if (tab) tab.downloads.unshift(...existing.downloads); + else + void NodeFSP.rm(downloadDir(existing), { recursive: true, force: true }).catch(constVoid); }).pipe(Effect.ignore), ), Effect.forkScoped, From 00cb6da90a653684c1516073e260f8372884eb80 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:42:32 -0700 Subject: [PATCH 3/6] fix(server): stalled browser viewers still learn to reconnect Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 7 ++++++- apps/server/src/preview/ServerBrowser.ts | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 973929c9bccb7..8fff8c3865170 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -1572,7 +1572,7 @@ it.live("a desktop page that comes back reconnects without waiting for a viewer" it.live("an agent tab that went headless moves onto the desktop page that mounts late", () => Effect.scoped( Effect.gen(function* () { - const { broker, tabId } = yield* ready; + const { browser, broker, tabId } = yield* ready; const headless = contexts[0]!.page; // The agent downloaded a file on the headless page; Chromium writes it, here the test does. const saved = Promise.withResolvers(); @@ -1596,12 +1596,17 @@ it.live("an agent tab that went headless moves onto the desktop page that mounts yield* Effect.yieldNow; before = yield* status(); } + // A remote viewer streams the headless page and has stopped reading. + const viewer = yield* browser.attachViewer(viewerInput(tabId, false)); + stallViewer(viewer, { _tag: "viewport", width: 1, height: 1 }); while (desktopDetaches.listenerCount("attach") === 0) yield* Effect.yieldNow; // The user opens the thread after the attach wait ran out, so the desktop mounts its page now. desktopTabs.add(tabId); desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId }); while (desktopConnections.length === 0) yield* Effect.yieldNow; expect(headless.isClosed()).toBe(true); + // Even with a full queue, the viewer learns to reconnect to the new page. + expect(yield* Queue.take(viewer.output)).toEqual({ _tag: "reconnect" }); yield* broker.invoke({ scope, tabId, diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 21ce307b091ee..388672d12d0a8 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -2237,7 +2237,12 @@ const make = Effect.gen(function* () { if (next._tag === "frame") runFork(next.ack); // State a stalled viewer cannot miss replaces its backlog. It runs // synchronously so an older replacement can never land after a newer one. - else if (next._tag === "gone" || next._tag === "control" || next._tag === "fileChooser") { + else if ( + next._tag === "gone" || + next._tag === "reconnect" || + next._tag === "control" || + next._tag === "fileChooser" + ) { const dropped = Effect.runSyncExit(Queue.clear(output)); if (dropped._tag === "Failure") return; Queue.offerUnsafe(output, next); From f08aab016bbb9149bdb21ae1c1899de0b105da5a Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:47:36 -0700 Subject: [PATCH 4/6] fix(server): a hidden desktop tab no longer freezes the agent's browser tools A snapshot's screenshot waits for the page's next frame. A desktop tab whose thread is not open draws none, so the snapshot held the tab and every later action, even evaluate, waited until the user opened the thread. The capture now gives up after 5 s with an error that says why and what to do. Co-Authored-By: Claude Opus 5.5 --- .../src/preview/ServerBrowserPage.test.ts | 37 ++++++++++++++++++- apps/server/src/preview/ServerBrowserPage.ts | 27 +++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 9a4775bb47173..0a07102dd1829 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -5,7 +5,16 @@ import { type CDPSession, type Page, } from "playwright-core"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vite-plus/test"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vite-plus/test"; import { presentAsChrome } from "./ServerBrowserContexts.ts"; import * as ServerBrowserPage from "./ServerBrowserPage.ts"; @@ -57,6 +66,32 @@ describe("server browser element refs", () => { }; const repeatedRows = `
    ${Array.from({ length: 5 }, (_, i) => `
  • row ${i + 1}
    • `).join("")}
`; + it("gives up on a screenshot when the page draws no frame", async () => { + // A desktop tab whose thread is not open never answers a capture. + const stuck = { + send: (method: string, params?: object) => + method === "Page.captureScreenshot" + ? new Promise(() => {}) + : cdp.send(method as "Page.getLayoutMetrics", params), + } as unknown as CDPSession; + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const snapshot = ServerBrowserPage.snapshot({ + page, + cdp: stuck, + renderScale: 1, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + }); + const rejected = expect(snapshot).rejects.toThrow("did not draw a frame"); + await vi.advanceTimersByTimeAsync(5_000); + await rejected; + } finally { + vi.useRealTimers(); + } + }); + it("clicks the fifth repeated delete control without touching row one", async () => { await page.setContent(repeatedRows); const result = await takeSnapshot(); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 3df4901f469c6..e31edf6dfc2fd 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -175,6 +175,31 @@ export const captureViewport = async ( return data; }; +const SNAPSHOT_CAPTURE_TIMEOUT_MS = 5_000; + +/** + * A capture waits for the page's next frame. A desktop tab whose thread is not + * open draws none, so without a limit the snapshot holds the tab and every + * later action waits behind it. + */ +const captureWithinLimit = (capture: Promise) => { + void capture.catch(constVoid); + let timer: ReturnType | undefined; + const limit = new Promise((_resolve, reject) => { + timer = setTimeout( + () => + reject( + new ServerBrowserOperationError( + "PreviewAutomationExecutionError", + "The page did not draw a frame, so no screenshot was taken. In the desktop app, a tab only draws while the user has its thread open. Use preview_evaluate to read the page, or ask the user to open the thread.", + ), + ), + SNAPSHOT_CAPTURE_TIMEOUT_MS, + ); + }); + return Promise.race([capture, limit]).finally(() => clearTimeout(timer)); +}; + export const snapshot = async (input: { readonly page: Page; readonly cdp: CDPSession; @@ -196,7 +221,7 @@ export const snapshot = async (input: { > >, input.page.ariaSnapshot({ mode: "ai", boxes: true, timeout: DEFAULT_TIMEOUT_MS }), - captureViewport(input.page, input.cdp, { format: "png", scale }), + captureWithinLimit(captureViewport(input.page, input.cdp, { format: "png", scale })), ]); if (state.generation !== generation) { throw new ServerBrowserOperationError( From a0e166709fc600bfce5c5671bf48fd135f3a2483 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:51:22 -0700 Subject: [PATCH 5/6] fix(server): every page capture gives up when the page draws no frame Co-Authored-By: Claude Opus 5.5 --- .../src/preview/ServerBrowserPage.test.ts | 23 +++++++---- apps/server/src/preview/ServerBrowserPage.ts | 41 +++++++++++-------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 0a07102dd1829..e397a4a40556d 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -68,15 +68,17 @@ describe("server browser element refs", () => { it("gives up on a screenshot when the page draws no frame", async () => { // A desktop tab whose thread is not open never answers a capture. + let captures = 0; const stuck = { - send: (method: string, params?: object) => - method === "Page.captureScreenshot" - ? new Promise(() => {}) - : cdp.send(method as "Page.getLayoutMetrics", params), + send: (method: string, params?: object) => { + if (method !== "Page.captureScreenshot") + return cdp.send(method as "Page.getLayoutMetrics", params); + captures += 1; + return new Promise(() => {}); + }, } as unknown as CDPSession; - vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); - try { - const snapshot = ServerBrowserPage.snapshot({ + const snapshotOf = () => + ServerBrowserPage.snapshot({ page, cdp: stuck, renderScale: 1, @@ -84,9 +86,14 @@ describe("server browser element refs", () => { networkEntries: [], actionTimeline: [], }); - const rejected = expect(snapshot).rejects.toThrow("did not draw a frame"); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const rejected = expect(snapshotOf()).rejects.toThrow("did not draw a frame"); await vi.advanceTimersByTimeAsync(5_000); await rejected; + // Chromium still holds the first capture, so a retry fails without starting another. + await expect(snapshotOf()).rejects.toThrow("did not draw a frame"); + expect(captures).toBe(1); } finally { vi.useRealTimers(); } diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index e31edf6dfc2fd..7ace8ecb79485 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -150,8 +150,13 @@ const SNAPSHOT_SCRIPT = `(() => { }; })()`; -// Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets. -export const captureViewport = async ( +const CAPTURE_TIMEOUT_MS = 5_000; +const NO_FRAME_MESSAGE = + "The page did not draw a frame, so no screenshot was taken. In the desktop app, a tab only draws while the user has its thread open. Use preview_evaluate to read the page, or ask the user to open the thread."; +/** Captures Chromium has not answered yet; a page only draws one at a time. */ +const pendingCaptures = new WeakSet(); + +const captureFrame = async ( page: Page, cdp: CDPSession, options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, @@ -175,27 +180,27 @@ export const captureViewport = async ( return data; }; -const SNAPSHOT_CAPTURE_TIMEOUT_MS = 5_000; - /** + * Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets. * A capture waits for the page's next frame. A desktop tab whose thread is not - * open draws none, so without a limit the snapshot holds the tab and every - * later action waits behind it. + * open draws none, so the capture gives up after a limit instead of holding + * the tab. A new capture is refused until Chromium answers the old one, which + * still owns the page's capture viewport. */ -const captureWithinLimit = (capture: Promise) => { +export const captureViewport = ( + page: Page, + cdp: CDPSession, + options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, +): Promise => { + const noFrame = () => + new ServerBrowserOperationError("PreviewAutomationExecutionError", NO_FRAME_MESSAGE); + if (pendingCaptures.has(page)) return Promise.reject(noFrame()); + pendingCaptures.add(page); + const capture = captureFrame(page, cdp, options).finally(() => pendingCaptures.delete(page)); void capture.catch(constVoid); let timer: ReturnType | undefined; const limit = new Promise((_resolve, reject) => { - timer = setTimeout( - () => - reject( - new ServerBrowserOperationError( - "PreviewAutomationExecutionError", - "The page did not draw a frame, so no screenshot was taken. In the desktop app, a tab only draws while the user has its thread open. Use preview_evaluate to read the page, or ask the user to open the thread.", - ), - ), - SNAPSHOT_CAPTURE_TIMEOUT_MS, - ); + timer = setTimeout(() => reject(noFrame()), CAPTURE_TIMEOUT_MS); }); return Promise.race([capture, limit]).finally(() => clearTimeout(timer)); }; @@ -221,7 +226,7 @@ export const snapshot = async (input: { > >, input.page.ariaSnapshot({ mode: "ai", boxes: true, timeout: DEFAULT_TIMEOUT_MS }), - captureWithinLimit(captureViewport(input.page, input.cdp, { format: "png", scale })), + captureViewport(input.page, input.cdp, { format: "png", scale }), ]); if (state.generation !== generation) { throw new ServerBrowserOperationError( From 8a3f2f7d0b10d3eba8dfb323d60459f31b819b32 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:58:41 -0700 Subject: [PATCH 6/6] fix(desktop): a hidden browser tab draws while the agent captures it, until the turn ends When the server asks for a screenshot of a desktop tab, the desktop now tells the web app, which keeps that webview drawing (the same lease recording uses) until the thread's turn ends. Before, a tab in a thread the user was not viewing sat offscreen and drew nothing, so agent screenshots never finished. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/ipc/channels.ts | 1 + apps/desktop/src/ipc/methods/preview.ts | 3 ++ apps/desktop/src/preload.ts | 8 +++ .../src/preview/DesktopBrowserHost.test.ts | 21 ++++++++ .../desktop/src/preview/DesktopBrowserHost.ts | 8 +++ apps/desktop/src/preview/Manager.ts | 50 +++++++++++++++---- apps/web/src/browser/ElectronBrowserHost.tsx | 9 ++++ .../src/browser/agentFrameRequests.test.ts | 42 ++++++++++++++++ apps/web/src/browser/agentFrameRequests.ts | 30 +++++++++++ packages/contracts/src/ipc.ts | 2 + 10 files changed, 164 insertions(+), 10 deletions(-) create mode 100644 apps/web/src/browser/agentFrameRequests.test.ts create mode 100644 apps/web/src/browser/agentFrameRequests.ts diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index a6b617385b578..a8f07bac29b1e 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -111,6 +111,7 @@ export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save"; export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame"; export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change"; export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event"; +export const PREVIEW_FRAME_REQUEST_CHANNEL = "desktop:preview-frame-request"; export const PREVIEW_OPEN_LINK_CHANNEL = "desktop:preview-open-link"; export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 94c56464f7c47..980cffdc855c3 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -54,6 +54,9 @@ export const installPreviewEventForwarding = Effect.fn( yield* manager.subscribePointerEvents((event) => electronWindow.sendAll(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, event), ); + yield* manager.subscribeFrameRequests((tabId) => + electronWindow.sendAll(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, tabId), + ); yield* manager.subscribeOpenLinks((event) => electronWindow.sendAll(IpcChannels.PREVIEW_OPEN_LINK_CHANNEL, event), ); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index efd49ffa38648..90136c5703a12 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -429,6 +429,14 @@ contextBridge.exposeInMainWorld("desktopBridge", { return () => ipcRenderer.removeListener(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, wrappedListener); }, + onFrameRequest: (listener) => { + const wrappedListener = (_event: Electron.IpcRendererEvent, tabId: unknown) => { + if (typeof tabId === "string") listener(tabId); + }; + ipcRenderer.on(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener); + return () => + ipcRenderer.removeListener(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener); + }, onOpenLink: (listener) => { const wrappedListener = (_event: Electron.IpcRendererEvent, linkEvent: unknown) => { if (typeof linkEvent !== "object" || linkEvent === null) return; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 81ea040701f0e..c1d3a6510e30f 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -148,4 +148,25 @@ describe("DesktopBrowserHost", () => { expect(host.humanStartedDownload(debuggee.tab.webContents)).toBe(false); }), ); + + it.effect("asks the tab to draw when the server waits for a capture", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + host.attach(key, makeDebuggee().tab); + const requests = yield* host.frameRequests.pipe( + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Effect.yieldNow; + const command = (id: number, method: string) => + host.handleCommandLine( + encodeJson({ type: "cdp", ...key, message: encodeJson({ id, method, params: {} }) }), + ); + // Reading the page needs no frame; a capture waits for the next one. + yield* command(1, "DOM.getDocument"); + yield* command(2, "Page.captureScreenshot"); + expect(yield* Fiber.join(requests)).toEqual([key]); + }).pipe(Effect.scoped), + ); }); diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index cfb1ff753c470..1d8cfb902879b 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -31,6 +31,8 @@ const lineEncoder = new TextEncoder(); /** CDP commands through which an agent clicks, types, or navigates the page. */ const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; +/** A capture waits for the page's next frame, which a hidden tab only draws on request. */ +const CAPTURE_COMMAND = /"method":"Page\.captureScreenshot"/; /** How long after an agent's input a download still counts as the agent's. */ const AGENT_DOWNLOAD_WINDOW_MS = 5_000; @@ -93,6 +95,8 @@ export class DesktopBrowserHost extends Context.Service< readonly x: number; readonly y: number; }>; + /** Server tabs whose page the server is waiting to capture, so they must draw. */ + readonly frameRequests: Stream.Stream; } >()("@t3tools/desktop/preview/DesktopBrowserHost") {} @@ -104,6 +108,7 @@ export const make = Effect.gen(function* () { readonly x: number; readonly y: number; }>(16); + const frameRequests = yield* PubSub.sliding(16); const runFork = Effect.runForkWith(yield* Effect.context()); const tabs = new Map(); const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event)); @@ -204,6 +209,8 @@ export const make = Effect.gen(function* () { return; } if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now(); + if (CAPTURE_COMMAND.test(command.value.message)) + runFork(PubSub.publish(frameRequests, tab.key)); relayFor(tab).receive(command.value.message); }); @@ -226,6 +233,7 @@ export const make = Effect.gen(function* () { return DesktopBrowserHost.of({ pointers: Stream.fromPubSub(pointers), + frameRequests: Stream.fromPubSub(frameRequests), // Subscribes before announcing, so no attach falls between the two. events: Stream.unwrap( Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index aa36055452eb8..ddf1c437238e0 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -415,6 +415,8 @@ type RecordingInputListener = (event: DesktopPreviewRecordingInputEvent) => Effe type PointerEventListener = (event: DesktopPreviewPointerEvent) => Effect.Effect; +type FrameRequestListener = (tabId: string) => Effect.Effect; + type OpenLinkListener = (event: DesktopPreviewOpenLinkEvent) => Effect.Effect; /** @@ -562,6 +564,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const attachedRef = yield* Ref.make>(new Map()); const listenersRef = yield* Ref.make>(new Set()); const pointerEventListenersRef = yield* Ref.make>(new Set()); + const frameRequestListenersRef = yield* Ref.make>(new Set()); const openLinkListenersRef = yield* Ref.make>(new Set()); const recordingInputListenersRef = yield* Ref.make>( new Set(), @@ -838,6 +841,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function | "recording-frame" | "recording-input" | "pointer-event" + | "frame-request" | "open-link", tabId: string, delivery: () => Effect.Effect, @@ -2411,7 +2415,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return; } if (afterAttach.serverTab) { - yield* listenForAgentPointers; + yield* listenForAgentSignals; browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger }); } if (afterAttach.colorScheme !== "system") { @@ -3343,17 +3347,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { discard: true }, ); }); - // Listening starts with the first tab of the desktop's own server. - let pointersStarted = false; - const listenForAgentPointers = Effect.suspend(() => { - if (pointersStarted) return Effect.void; - pointersStarted = true; - return browserHost.pointers.pipe( - Stream.runForEach(emitAgentPointer), - Effect.forkIn(parentScope), - Effect.asVoid, + /** Tells the renderer that the server waits for a frame of this tab, so it must draw. */ + const emitFrameRequest = Effect.fn("PreviewManager.emitFrameRequest")(function* (key: { + readonly threadId: string; + readonly tabId: string; + }) { + const tab = [...(yield* SynchronizedRef.get(tabsRef)).values()].find( + (candidate) => + candidate.serverTab?.threadId === key.threadId && candidate.serverTab.tabId === key.tabId, + ); + if (!tab) return; + const listeners = yield* Ref.get(frameRequestListenersRef); + yield* Effect.forEach( + listeners, + (listener) => deliverEvent("frame-request", tab.tabId, () => listener(tab.tabId)), + { discard: true }, ); }); + // Listening starts with the first tab of the desktop's own server. + let agentSignalsStarted = false; + const listenForAgentSignals = Effect.suspend(() => { + if (agentSignalsStarted) return Effect.void; + agentSignalsStarted = true; + return Effect.all([ + browserHost.pointers.pipe(Stream.runForEach(emitAgentPointer), Effect.forkIn(parentScope)), + browserHost.frameRequests.pipe( + Stream.runForEach(emitFrameRequest), + Effect.forkIn(parentScope), + ), + ]).pipe(Effect.asVoid); + }); const emitOpenLink = Effect.fn("PreviewManager.emitOpenLink")(function* ( event: DesktopPreviewOpenLinkEvent, @@ -3386,6 +3409,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function [ Ref.set(listenersRef, new Set()), Ref.set(pointerEventListenersRef, new Set()), + Ref.set(frameRequestListenersRef, new Set()), Ref.set(openLinkListenersRef, new Set()), Ref.set(recordingFrameListenersRef, new Set()), Ref.set(recordingInputListenersRef, new Set()), @@ -3432,6 +3456,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function stopRecording, subscribePointerEvents: (listener: PointerEventListener) => subscribe(pointerEventListenersRef, listener), + subscribeFrameRequests: (listener: FrameRequestListener) => + subscribe(frameRequestListenersRef, listener), subscribeOpenLinks: (listener: OpenLinkListener) => subscribe(openLinkListenersRef, listener), subscribeRecordingInputs: (listener: RecordingInputListener) => subscribe(recordingInputListenersRef, listener), @@ -3674,6 +3700,9 @@ export class PreviewManager extends Context.Service< readonly subscribePointerEvents: ( listener: PointerEventListener, ) => Effect.Effect; + readonly subscribeFrameRequests: ( + listener: FrameRequestListener, + ) => Effect.Effect; readonly subscribeOpenLinks: ( listener: OpenLinkListener, ) => Effect.Effect; @@ -3819,6 +3848,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { saveRecording: operations.saveRecording, subscribeStateChanges: operations.subscribeStateChanges, subscribePointerEvents: operations.subscribePointerEvents, + subscribeFrameRequests: operations.subscribeFrameRequests, subscribeOpenLinks: operations.subscribeOpenLinks, subscribeRecordingFrames: operations.subscribeRecordingFrames, subscribeRecordingInputs: operations.subscribeRecordingInputs, diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index b076a30a5221f..2e602201d6ec9 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -15,6 +15,7 @@ import { previewEnvironment } from "~/state/preview"; import { useEnvironmentScope } from "~/state/session"; import { useAtomCommand } from "~/state/use-atom-command"; +import { holdFramesForTurn } from "./agentFrameRequests"; import { readPreviewAnnotationTheme } from "./annotationTheme"; import { useBrowserDefaults } from "./browserDefaults"; import { useBrowserPointerStore } from "./browserPointerStore"; @@ -106,6 +107,14 @@ export function ElectronBrowserHost() { sessions.map((session) => [session.runtimeTabId, session]), ); }, [sessions]); + useEffect(() => { + const preview = window.desktopBridge?.preview; + if (!preview) return; + return preview.onFrameRequest((tabId) => { + const source = sessionByRuntimeTabId.current.get(tabId); + if (source) holdFramesForTurn(tabId, source.threadRef); + }); + }, []); useEffect(() => { const preview = window.desktopBridge?.preview; if (!preview) return; diff --git a/apps/web/src/browser/agentFrameRequests.test.ts b/apps/web/src/browser/agentFrameRequests.test.ts new file mode 100644 index 0000000000000..e7f161fcd3217 --- /dev/null +++ b/apps/web/src/browser/agentFrameRequests.test.ts @@ -0,0 +1,42 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { Atom } from "effect/reactivity"; +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { appAtomRegistry } from "~/rpc/atomRegistry"; + +import { holdFramesForTurn } from "./agentFrameRequests"; +import { useBrowserSurfaceStore } from "./browserSurfaceStore"; + +const shell = vi.hoisted(() => ({ atom: null as unknown })); +vi.mock("~/state/threads", () => ({ + environmentThreadShells: { threadShellAtom: () => shell.atom }, +})); + +type Shell = { readonly runtime: { readonly status: string } | null }; +const threadRef = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), +}; +const shellAtom = Atom.make(null).pipe(Atom.keepAlive); +shell.atom = shellAtom; +const setStatus = (status: string) => appAtomRegistry.set(shellAtom, { runtime: { status } }); +const drawing = (tabId: string) => useBrowserSurfaceStore.getState().activityByTabId[tabId] ?? 0; + +describe("holdFramesForTurn", () => { + beforeEach(() => setStatus("idle")); + + it("keeps a hidden tab drawing until the thread's turn ends", () => { + setStatus("running"); + holdFramesForTurn("tab-1", threadRef); + // Later captures in the same turn reuse the lease. + holdFramesForTurn("tab-1", threadRef); + expect(drawing("tab-1")).toBe(1); + setStatus("idle"); + expect(drawing("tab-1")).toBe(0); + }); + + it("does not draw for a capture outside a turn", () => { + holdFramesForTurn("tab-2", threadRef); + expect(drawing("tab-2")).toBe(0); + }); +}); diff --git a/apps/web/src/browser/agentFrameRequests.ts b/apps/web/src/browser/agentFrameRequests.ts new file mode 100644 index 0000000000000..6fc060e6c5309 --- /dev/null +++ b/apps/web/src/browser/agentFrameRequests.ts @@ -0,0 +1,30 @@ +import { threadRuntimeIsActive } from "@t3tools/client-runtime/state/models"; +import type { ScopedThreadRef } from "@t3tools/contracts"; + +import { appAtomRegistry } from "~/rpc/atomRegistry"; +import { environmentThreadShells } from "~/state/threads"; + +import { acquireBrowserSurfaceActivity } from "./browserSurfaceStore"; + +const leases = new Map void>(); + +/** + * Keeps a hidden tab drawing after the server asked for one of its frames, until + * the thread's turn ends. A tab parked offscreen draws nothing, so an agent's + * screenshot would wait forever. Later requests in the same turn reuse the lease. + */ +export function holdFramesForTurn(tabId: string, threadRef: ScopedThreadRef): void { + if (leases.has(tabId)) return; + const shellAtom = environmentThreadShells.threadShellAtom(threadRef); + if (!threadRuntimeIsActive(appAtomRegistry.get(shellAtom)?.runtime)) return; + const releaseActivity = acquireBrowserSurfaceActivity(tabId); + const release = () => { + leases.delete(tabId); + unsubscribe(); + releaseActivity(); + }; + const unsubscribe = appAtomRegistry.subscribe(shellAtom, (shell) => { + if (!threadRuntimeIsActive(shell?.runtime)) release(); + }); + leases.set(tabId, release); +} diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index e521c4c37271c..7b86c3131365c 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1352,6 +1352,8 @@ export interface DesktopPreviewBridge { }; onStateChange: (listener: (tabId: string, state: DesktopPreviewTabState) => void) => () => void; onPointerEvent: (listener: (event: DesktopPreviewPointerEvent) => void) => () => void; + /** The server waits for a frame of this hidden tab, so it should draw until the turn ends. */ + onFrameRequest: (listener: (tabId: string) => void) => () => void; onOpenLink: (listener: (event: DesktopPreviewOpenLinkEvent) => void) => () => void; }