diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index a6b617385b578..a8f07bac29b1e 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -111,6 +111,7 @@ export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save"; export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame"; export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change"; export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event"; +export const PREVIEW_FRAME_REQUEST_CHANNEL = "desktop:preview-frame-request"; export const PREVIEW_OPEN_LINK_CHANNEL = "desktop:preview-open-link"; export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 94c56464f7c47..980cffdc855c3 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -54,6 +54,9 @@ export const installPreviewEventForwarding = Effect.fn( yield* manager.subscribePointerEvents((event) => electronWindow.sendAll(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, event), ); + yield* manager.subscribeFrameRequests((tabId) => + electronWindow.sendAll(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, tabId), + ); yield* manager.subscribeOpenLinks((event) => electronWindow.sendAll(IpcChannels.PREVIEW_OPEN_LINK_CHANNEL, event), ); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index efd49ffa38648..90136c5703a12 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -429,6 +429,14 @@ contextBridge.exposeInMainWorld("desktopBridge", { return () => ipcRenderer.removeListener(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, wrappedListener); }, + onFrameRequest: (listener) => { + const wrappedListener = (_event: Electron.IpcRendererEvent, tabId: unknown) => { + if (typeof tabId === "string") listener(tabId); + }; + ipcRenderer.on(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener); + return () => + ipcRenderer.removeListener(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener); + }, onOpenLink: (listener) => { const wrappedListener = (_event: Electron.IpcRendererEvent, linkEvent: unknown) => { if (typeof linkEvent !== "object" || linkEvent === null) return; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 81ea040701f0e..c1d3a6510e30f 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -148,4 +148,25 @@ describe("DesktopBrowserHost", () => { expect(host.humanStartedDownload(debuggee.tab.webContents)).toBe(false); }), ); + + it.effect("asks the tab to draw when the server waits for a capture", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + host.attach(key, makeDebuggee().tab); + const requests = yield* host.frameRequests.pipe( + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Effect.yieldNow; + const command = (id: number, method: string) => + host.handleCommandLine( + encodeJson({ type: "cdp", ...key, message: encodeJson({ id, method, params: {} }) }), + ); + // Reading the page needs no frame; a capture waits for the next one. + yield* command(1, "DOM.getDocument"); + yield* command(2, "Page.captureScreenshot"); + expect(yield* Fiber.join(requests)).toEqual([key]); + }).pipe(Effect.scoped), + ); }); diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index cfb1ff753c470..1d8cfb902879b 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -31,6 +31,8 @@ const lineEncoder = new TextEncoder(); /** CDP commands through which an agent clicks, types, or navigates the page. */ const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; +/** A capture waits for the page's next frame, which a hidden tab only draws on request. */ +const CAPTURE_COMMAND = /"method":"Page\.captureScreenshot"/; /** How long after an agent's input a download still counts as the agent's. */ const AGENT_DOWNLOAD_WINDOW_MS = 5_000; @@ -93,6 +95,8 @@ export class DesktopBrowserHost extends Context.Service< readonly x: number; readonly y: number; }>; + /** Server tabs whose page the server is waiting to capture, so they must draw. */ + readonly frameRequests: Stream.Stream; } >()("@t3tools/desktop/preview/DesktopBrowserHost") {} @@ -104,6 +108,7 @@ export const make = Effect.gen(function* () { readonly x: number; readonly y: number; }>(16); + const frameRequests = yield* PubSub.sliding(16); const runFork = Effect.runForkWith(yield* Effect.context()); const tabs = new Map(); const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event)); @@ -204,6 +209,8 @@ export const make = Effect.gen(function* () { return; } if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now(); + if (CAPTURE_COMMAND.test(command.value.message)) + runFork(PubSub.publish(frameRequests, tab.key)); relayFor(tab).receive(command.value.message); }); @@ -226,6 +233,7 @@ export const make = Effect.gen(function* () { return DesktopBrowserHost.of({ pointers: Stream.fromPubSub(pointers), + frameRequests: Stream.fromPubSub(frameRequests), // Subscribes before announcing, so no attach falls between the two. events: Stream.unwrap( Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index aa36055452eb8..ddf1c437238e0 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -415,6 +415,8 @@ type RecordingInputListener = (event: DesktopPreviewRecordingInputEvent) => Effe type PointerEventListener = (event: DesktopPreviewPointerEvent) => Effect.Effect; +type FrameRequestListener = (tabId: string) => Effect.Effect; + type OpenLinkListener = (event: DesktopPreviewOpenLinkEvent) => Effect.Effect; /** @@ -562,6 +564,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const attachedRef = yield* Ref.make>(new Map()); const listenersRef = yield* Ref.make>(new Set()); const pointerEventListenersRef = yield* Ref.make>(new Set()); + const frameRequestListenersRef = yield* Ref.make>(new Set()); const openLinkListenersRef = yield* Ref.make>(new Set()); const recordingInputListenersRef = yield* Ref.make>( new Set(), @@ -838,6 +841,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function | "recording-frame" | "recording-input" | "pointer-event" + | "frame-request" | "open-link", tabId: string, delivery: () => Effect.Effect, @@ -2411,7 +2415,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return; } if (afterAttach.serverTab) { - yield* listenForAgentPointers; + yield* listenForAgentSignals; browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger }); } if (afterAttach.colorScheme !== "system") { @@ -3343,17 +3347,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { discard: true }, ); }); - // Listening starts with the first tab of the desktop's own server. - let pointersStarted = false; - const listenForAgentPointers = Effect.suspend(() => { - if (pointersStarted) return Effect.void; - pointersStarted = true; - return browserHost.pointers.pipe( - Stream.runForEach(emitAgentPointer), - Effect.forkIn(parentScope), - Effect.asVoid, + /** Tells the renderer that the server waits for a frame of this tab, so it must draw. */ + const emitFrameRequest = Effect.fn("PreviewManager.emitFrameRequest")(function* (key: { + readonly threadId: string; + readonly tabId: string; + }) { + const tab = [...(yield* SynchronizedRef.get(tabsRef)).values()].find( + (candidate) => + candidate.serverTab?.threadId === key.threadId && candidate.serverTab.tabId === key.tabId, + ); + if (!tab) return; + const listeners = yield* Ref.get(frameRequestListenersRef); + yield* Effect.forEach( + listeners, + (listener) => deliverEvent("frame-request", tab.tabId, () => listener(tab.tabId)), + { discard: true }, ); }); + // Listening starts with the first tab of the desktop's own server. + let agentSignalsStarted = false; + const listenForAgentSignals = Effect.suspend(() => { + if (agentSignalsStarted) return Effect.void; + agentSignalsStarted = true; + return Effect.all([ + browserHost.pointers.pipe(Stream.runForEach(emitAgentPointer), Effect.forkIn(parentScope)), + browserHost.frameRequests.pipe( + Stream.runForEach(emitFrameRequest), + Effect.forkIn(parentScope), + ), + ]).pipe(Effect.asVoid); + }); const emitOpenLink = Effect.fn("PreviewManager.emitOpenLink")(function* ( event: DesktopPreviewOpenLinkEvent, @@ -3386,6 +3409,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function [ Ref.set(listenersRef, new Set()), Ref.set(pointerEventListenersRef, new Set()), + Ref.set(frameRequestListenersRef, new Set()), Ref.set(openLinkListenersRef, new Set()), Ref.set(recordingFrameListenersRef, new Set()), Ref.set(recordingInputListenersRef, new Set()), @@ -3432,6 +3456,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function stopRecording, subscribePointerEvents: (listener: PointerEventListener) => subscribe(pointerEventListenersRef, listener), + subscribeFrameRequests: (listener: FrameRequestListener) => + subscribe(frameRequestListenersRef, listener), subscribeOpenLinks: (listener: OpenLinkListener) => subscribe(openLinkListenersRef, listener), subscribeRecordingInputs: (listener: RecordingInputListener) => subscribe(recordingInputListenersRef, listener), @@ -3674,6 +3700,9 @@ export class PreviewManager extends Context.Service< readonly subscribePointerEvents: ( listener: PointerEventListener, ) => Effect.Effect; + readonly subscribeFrameRequests: ( + listener: FrameRequestListener, + ) => Effect.Effect; readonly subscribeOpenLinks: ( listener: OpenLinkListener, ) => Effect.Effect; @@ -3819,6 +3848,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { saveRecording: operations.saveRecording, subscribeStateChanges: operations.subscribeStateChanges, subscribePointerEvents: operations.subscribePointerEvents, + subscribeFrameRequests: operations.subscribeFrameRequests, subscribeOpenLinks: operations.subscribeOpenLinks, subscribeRecordingFrames: operations.subscribeRecordingFrames, subscribeRecordingInputs: operations.subscribeRecordingInputs, diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 385223a42d397..8fff8c3865170 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -1568,3 +1568,60 @@ it.live("a desktop page that comes back reconnects without waiting for a viewer" }), ).pipe(Effect.provide(layer)), ); + +it.live("an agent tab that went headless moves onto the desktop page that mounts late", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, broker, tabId } = yield* ready; + const headless = contexts[0]!.page; + // The agent downloaded a file on the headless page; Chromium writes it, here the test does. + const saved = Promise.withResolvers(); + const written = Promise.withResolvers(); + headless.emit("download", { + failure: async () => null, + saveAs: (path: string) => { + saved.resolve(path); + return written.promise; + }, + suggestedFilename: () => "report.csv", + url: () => "blob:http://localhost:5173/1", + }); + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString(yield* Effect.promise(() => saved.promise), "a,b"); + written.resolve(); + const status = () => + broker.invoke({ scope, tabId, operation: "status", input: {} }); + let before = yield* status(); + while (!before.downloads?.length) { + yield* Effect.yieldNow; + before = yield* status(); + } + // A remote viewer streams the headless page and has stopped reading. + const viewer = yield* browser.attachViewer(viewerInput(tabId, false)); + stallViewer(viewer, { _tag: "viewport", width: 1, height: 1 }); + while (desktopDetaches.listenerCount("attach") === 0) yield* Effect.yieldNow; + // The user opens the thread after the attach wait ran out, so the desktop mounts its page now. + desktopTabs.add(tabId); + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId }); + while (desktopConnections.length === 0) yield* Effect.yieldNow; + expect(headless.isClosed()).toBe(true); + // Even with a full queue, the viewer learns to reconnect to the new page. + expect(yield* Queue.take(viewer.output)).toEqual({ _tag: "reconnect" }); + yield* broker.invoke({ + scope, + tabId, + operation: "navigate", + input: { url: "http://localhost:5173/next" }, + }); + // The agent now drives the page the user sees. + const desktopPage = desktopConnections[0]!.context.page; + expect(desktopPage.goto.mock.calls.map(([url]) => url)).toContain( + "http://localhost:5173/next", + ); + // The session stayed open, so the agent's download is still there. + const after = yield* status(); + expect(after.downloads).toEqual(before.downloads); + expect(yield* fs.exists(after.downloads?.[0]?.path ?? "")).toBe(true); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 939d112c49081..388672d12d0a8 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -683,14 +683,16 @@ const make = Effect.gen(function* () { /** Whether the preview session for a tab still exists. */ const sessionOpen = (tab: ServerTab) => !closedSessions.has(tabKey(tab.threadId, tab.tabId)); - const dropTab = (tab: ServerTab, closeSession: boolean) => { + /** `handOff` keeps the session for a new page, so viewers reconnect to it. */ + const dropTab = (tab: ServerTab, closeSession: boolean, handOff = false) => { const key = tabKey(tab.threadId, tab.tabId); if (tabs.get(key) !== tab) return; tabs.delete(key); tab.closing = true; clearAbortedNavigation(tab); // A desktop page outlives the connection unless its session closed with it. - const end = tab.desktop && !closeSession && sessionOpen(tab) ? "reconnect" : "gone"; + const end = + (tab.desktop || handOff) && !closeSession && sessionOpen(tab) ? "reconnect" : "gone"; for (const viewer of tab.viewers) viewer.push({ _tag: end }); void tab.control.close().catch(constVoid); // The desktop owns its page; letting go only ends this connection. @@ -698,7 +700,9 @@ const make = Effect.gen(function* () { else void tab.page.close().catch(constVoid); if (tab.isolatedContext) void tab.page.context().close().catch(constVoid); void tab.recording?.encoder.close().catch(constVoid); - void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid); + // A handed-off tab keeps its downloads; the folder is keyed by tab, not page. + if (!handOff) + void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid); reportLiveTabs(); if (closeSession) { runFork(manager.close({ threadId: tab.threadId, tabId: tab.tabId }).pipe(Effect.ignore)); @@ -2233,7 +2237,12 @@ const make = Effect.gen(function* () { if (next._tag === "frame") runFork(next.ack); // State a stalled viewer cannot miss replaces its backlog. It runs // synchronously so an older replacement can never land after a newer one. - else if (next._tag === "gone" || next._tag === "control" || next._tag === "fileChooser") { + else if ( + next._tag === "gone" || + next._tag === "reconnect" || + next._tag === "control" || + next._tag === "fileChooser" + ) { const dropped = Effect.runSyncExit(Queue.clear(output)); if (dropped._tag === "Failure") return; Queue.offerUnsafe(output, next); @@ -2408,15 +2417,28 @@ const make = Effect.gen(function* () { // The page came back, for example after its DevTools closed. Reconnect now // rather than on the next viewer or agent, or the tab's URL and title stop // reaching every client in the meantime. + // A tab that went headless before the desktop mounted it (its thread was in + // the background) moves onto the desktop's page. Otherwise the user and the + // agent see two different pages under one tab. yield* desktopChannel.attached.pipe( Stream.runForEach((key) => Effect.gen(function* () { - if (tabs.has(tabKey(key.threadId, key.tabId))) return; + const pending = pendingTabs.get(tabKey(key.threadId, key.tabId)); + if (pending) yield* Effect.promise(() => pending.catch(constVoid)); + const existing = tabs.get(tabKey(key.threadId, key.tabId)); + if (existing?.desktop) return; + if (existing) dropTab(existing, false, true); const { sessions } = yield* manager.list({ threadId: ThreadId.make(key.threadId) }); const snapshot = sessions.find( (session) => session.tabId === key.tabId && session.runtime === "server", ); - if (snapshot) yield* Effect.promise(() => ensureTab(snapshot).catch(constVoid)); + const tab = snapshot + ? yield* Effect.promise(() => ensureTab(snapshot).catch(() => null)) + : null; + if (!existing) return; + if (tab) tab.downloads.unshift(...existing.downloads); + else + void NodeFSP.rm(downloadDir(existing), { recursive: true, force: true }).catch(constVoid); }).pipe(Effect.ignore), ), Effect.forkScoped, diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 9a4775bb47173..e397a4a40556d 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -5,7 +5,16 @@ import { type CDPSession, type Page, } from "playwright-core"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vite-plus/test"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vite-plus/test"; import { presentAsChrome } from "./ServerBrowserContexts.ts"; import * as ServerBrowserPage from "./ServerBrowserPage.ts"; @@ -57,6 +66,39 @@ describe("server browser element refs", () => { }; const repeatedRows = `
    ${Array.from({ length: 5 }, (_, i) => `
  • row ${i + 1}
    • `).join("")}
`; + it("gives up on a screenshot when the page draws no frame", async () => { + // A desktop tab whose thread is not open never answers a capture. + let captures = 0; + const stuck = { + send: (method: string, params?: object) => { + if (method !== "Page.captureScreenshot") + return cdp.send(method as "Page.getLayoutMetrics", params); + captures += 1; + return new Promise(() => {}); + }, + } as unknown as CDPSession; + const snapshotOf = () => + ServerBrowserPage.snapshot({ + page, + cdp: stuck, + renderScale: 1, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const rejected = expect(snapshotOf()).rejects.toThrow("did not draw a frame"); + await vi.advanceTimersByTimeAsync(5_000); + await rejected; + // Chromium still holds the first capture, so a retry fails without starting another. + await expect(snapshotOf()).rejects.toThrow("did not draw a frame"); + expect(captures).toBe(1); + } finally { + vi.useRealTimers(); + } + }); + it("clicks the fifth repeated delete control without touching row one", async () => { await page.setContent(repeatedRows); const result = await takeSnapshot(); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 3df4901f469c6..7ace8ecb79485 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -150,8 +150,13 @@ const SNAPSHOT_SCRIPT = `(() => { }; })()`; -// Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets. -export const captureViewport = async ( +const CAPTURE_TIMEOUT_MS = 5_000; +const NO_FRAME_MESSAGE = + "The page did not draw a frame, so no screenshot was taken. In the desktop app, a tab only draws while the user has its thread open. Use preview_evaluate to read the page, or ask the user to open the thread."; +/** Captures Chromium has not answered yet; a page only draws one at a time. */ +const pendingCaptures = new WeakSet(); + +const captureFrame = async ( page: Page, cdp: CDPSession, options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, @@ -175,6 +180,31 @@ export const captureViewport = async ( return data; }; +/** + * Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets. + * A capture waits for the page's next frame. A desktop tab whose thread is not + * open draws none, so the capture gives up after a limit instead of holding + * the tab. A new capture is refused until Chromium answers the old one, which + * still owns the page's capture viewport. + */ +export const captureViewport = ( + page: Page, + cdp: CDPSession, + options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, +): Promise => { + const noFrame = () => + new ServerBrowserOperationError("PreviewAutomationExecutionError", NO_FRAME_MESSAGE); + if (pendingCaptures.has(page)) return Promise.reject(noFrame()); + pendingCaptures.add(page); + const capture = captureFrame(page, cdp, options).finally(() => pendingCaptures.delete(page)); + void capture.catch(constVoid); + let timer: ReturnType | undefined; + const limit = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(noFrame()), CAPTURE_TIMEOUT_MS); + }); + return Promise.race([capture, limit]).finally(() => clearTimeout(timer)); +}; + export const snapshot = async (input: { readonly page: Page; readonly cdp: CDPSession; diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index b076a30a5221f..2e602201d6ec9 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -15,6 +15,7 @@ import { previewEnvironment } from "~/state/preview"; import { useEnvironmentScope } from "~/state/session"; import { useAtomCommand } from "~/state/use-atom-command"; +import { holdFramesForTurn } from "./agentFrameRequests"; import { readPreviewAnnotationTheme } from "./annotationTheme"; import { useBrowserDefaults } from "./browserDefaults"; import { useBrowserPointerStore } from "./browserPointerStore"; @@ -106,6 +107,14 @@ export function ElectronBrowserHost() { sessions.map((session) => [session.runtimeTabId, session]), ); }, [sessions]); + useEffect(() => { + const preview = window.desktopBridge?.preview; + if (!preview) return; + return preview.onFrameRequest((tabId) => { + const source = sessionByRuntimeTabId.current.get(tabId); + if (source) holdFramesForTurn(tabId, source.threadRef); + }); + }, []); useEffect(() => { const preview = window.desktopBridge?.preview; if (!preview) return; diff --git a/apps/web/src/browser/agentFrameRequests.test.ts b/apps/web/src/browser/agentFrameRequests.test.ts new file mode 100644 index 0000000000000..e7f161fcd3217 --- /dev/null +++ b/apps/web/src/browser/agentFrameRequests.test.ts @@ -0,0 +1,42 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { Atom } from "effect/reactivity"; +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { appAtomRegistry } from "~/rpc/atomRegistry"; + +import { holdFramesForTurn } from "./agentFrameRequests"; +import { useBrowserSurfaceStore } from "./browserSurfaceStore"; + +const shell = vi.hoisted(() => ({ atom: null as unknown })); +vi.mock("~/state/threads", () => ({ + environmentThreadShells: { threadShellAtom: () => shell.atom }, +})); + +type Shell = { readonly runtime: { readonly status: string } | null }; +const threadRef = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), +}; +const shellAtom = Atom.make(null).pipe(Atom.keepAlive); +shell.atom = shellAtom; +const setStatus = (status: string) => appAtomRegistry.set(shellAtom, { runtime: { status } }); +const drawing = (tabId: string) => useBrowserSurfaceStore.getState().activityByTabId[tabId] ?? 0; + +describe("holdFramesForTurn", () => { + beforeEach(() => setStatus("idle")); + + it("keeps a hidden tab drawing until the thread's turn ends", () => { + setStatus("running"); + holdFramesForTurn("tab-1", threadRef); + // Later captures in the same turn reuse the lease. + holdFramesForTurn("tab-1", threadRef); + expect(drawing("tab-1")).toBe(1); + setStatus("idle"); + expect(drawing("tab-1")).toBe(0); + }); + + it("does not draw for a capture outside a turn", () => { + holdFramesForTurn("tab-2", threadRef); + expect(drawing("tab-2")).toBe(0); + }); +}); diff --git a/apps/web/src/browser/agentFrameRequests.ts b/apps/web/src/browser/agentFrameRequests.ts new file mode 100644 index 0000000000000..6fc060e6c5309 --- /dev/null +++ b/apps/web/src/browser/agentFrameRequests.ts @@ -0,0 +1,30 @@ +import { threadRuntimeIsActive } from "@t3tools/client-runtime/state/models"; +import type { ScopedThreadRef } from "@t3tools/contracts"; + +import { appAtomRegistry } from "~/rpc/atomRegistry"; +import { environmentThreadShells } from "~/state/threads"; + +import { acquireBrowserSurfaceActivity } from "./browserSurfaceStore"; + +const leases = new Map void>(); + +/** + * Keeps a hidden tab drawing after the server asked for one of its frames, until + * the thread's turn ends. A tab parked offscreen draws nothing, so an agent's + * screenshot would wait forever. Later requests in the same turn reuse the lease. + */ +export function holdFramesForTurn(tabId: string, threadRef: ScopedThreadRef): void { + if (leases.has(tabId)) return; + const shellAtom = environmentThreadShells.threadShellAtom(threadRef); + if (!threadRuntimeIsActive(appAtomRegistry.get(shellAtom)?.runtime)) return; + const releaseActivity = acquireBrowserSurfaceActivity(tabId); + const release = () => { + leases.delete(tabId); + unsubscribe(); + releaseActivity(); + }; + const unsubscribe = appAtomRegistry.subscribe(shellAtom, (shell) => { + if (!threadRuntimeIsActive(shell?.runtime)) release(); + }); + leases.set(tabId, release); +} diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index e521c4c37271c..7b86c3131365c 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1352,6 +1352,8 @@ export interface DesktopPreviewBridge { }; onStateChange: (listener: (tabId: string, state: DesktopPreviewTabState) => void) => () => void; onPointerEvent: (listener: (event: DesktopPreviewPointerEvent) => void) => () => void; + /** The server waits for a frame of this hidden tab, so it should draw until the turn ends. */ + onFrameRequest: (listener: (tabId: string) => void) => () => void; onOpenLink: (listener: (event: DesktopPreviewOpenLinkEvent) => void) => () => void; }