From a62826c8e729faaa8bc51d4fdccc9e4b4035c19b Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:34:08 -0700 Subject: [PATCH 1/7] fix(server): agent browser tabs move to the desktop page when it mounts late A tab that the agent opens while its thread is in the background falls back to a headless page after 10 s. When the user opens the thread later, the desktop mounts its own page for the same tab, but the server kept driving the headless one. The user and the agent saw two different pages. On a late attach, drop the headless page and reconnect the tab to the desktop page. Stream viewers reconnect to the new page. Refs #16901 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 26 +++++++++++++++++++ apps/server/src/preview/ServerBrowser.ts | 15 ++++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 385223a42d397..de6f4f1b8c423 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -1568,3 +1568,29 @@ it.live("a desktop page that comes back reconnects without waiting for a viewer" }), ).pipe(Effect.provide(layer)), ); + +it.live("an agent tab that went headless moves onto the desktop page that mounts late", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const headless = contexts[0]!.page; + while (desktopDetaches.listenerCount("attach") === 0) yield* Effect.yieldNow; + // The user opens the thread after the attach wait ran out, so the desktop mounts its page now. + desktopTabs.add(tabId); + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId }); + while (desktopConnections.length === 0) yield* Effect.yieldNow; + expect(headless.isClosed()).toBe(true); + yield* broker.invoke({ + scope, + tabId, + operation: "navigate", + input: { url: "http://localhost:5173/next" }, + }); + // The agent now drives the page the user sees. + const desktopPage = desktopConnections[0]!.context.page; + expect(desktopPage.goto.mock.calls.map(([url]) => url)).toContain( + "http://localhost:5173/next", + ); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 939d112c49081..5807c8b67c7d0 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -683,14 +683,16 @@ const make = Effect.gen(function* () { /** Whether the preview session for a tab still exists. */ const sessionOpen = (tab: ServerTab) => !closedSessions.has(tabKey(tab.threadId, tab.tabId)); - const dropTab = (tab: ServerTab, closeSession: boolean) => { + /** `handOff` keeps the session for a new page, so viewers reconnect to it. */ + const dropTab = (tab: ServerTab, closeSession: boolean, handOff = false) => { const key = tabKey(tab.threadId, tab.tabId); if (tabs.get(key) !== tab) return; tabs.delete(key); tab.closing = true; clearAbortedNavigation(tab); // A desktop page outlives the connection unless its session closed with it. - const end = tab.desktop && !closeSession && sessionOpen(tab) ? "reconnect" : "gone"; + const end = + (tab.desktop || handOff) && !closeSession && sessionOpen(tab) ? "reconnect" : "gone"; for (const viewer of tab.viewers) viewer.push({ _tag: end }); void tab.control.close().catch(constVoid); // The desktop owns its page; letting go only ends this connection. @@ -2408,10 +2410,17 @@ const make = Effect.gen(function* () { // The page came back, for example after its DevTools closed. Reconnect now // rather than on the next viewer or agent, or the tab's URL and title stop // reaching every client in the meantime. + // A tab that went headless before the desktop mounted it (its thread was in + // the background) moves onto the desktop's page. Otherwise the user and the + // agent see two different pages under one tab. yield* desktopChannel.attached.pipe( Stream.runForEach((key) => Effect.gen(function* () { - if (tabs.has(tabKey(key.threadId, key.tabId))) return; + const pending = pendingTabs.get(tabKey(key.threadId, key.tabId)); + if (pending) yield* Effect.promise(() => pending.catch(constVoid)); + const existing = tabs.get(tabKey(key.threadId, key.tabId)); + if (existing?.desktop) return; + if (existing) dropTab(existing, false, true); const { sessions } = yield* manager.list({ threadId: ThreadId.make(key.threadId) }); const snapshot = sessions.find( (session) => session.tabId === key.tabId && session.runtime === "server", From fcf58f0468bbc4aed226b368fb44b231110fdd38 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:40:22 -0700 Subject: [PATCH 2/7] fix(server): keep an agent's downloads when its tab moves to the desktop page Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 26 +++++++++++++++++++ apps/server/src/preview/ServerBrowser.ts | 12 +++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index de6f4f1b8c423..973929c9bccb7 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -1574,6 +1574,28 @@ it.live("an agent tab that went headless moves onto the desktop page that mounts Effect.gen(function* () { const { broker, tabId } = yield* ready; const headless = contexts[0]!.page; + // The agent downloaded a file on the headless page; Chromium writes it, here the test does. + const saved = Promise.withResolvers(); + const written = Promise.withResolvers(); + headless.emit("download", { + failure: async () => null, + saveAs: (path: string) => { + saved.resolve(path); + return written.promise; + }, + suggestedFilename: () => "report.csv", + url: () => "blob:http://localhost:5173/1", + }); + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString(yield* Effect.promise(() => saved.promise), "a,b"); + written.resolve(); + const status = () => + broker.invoke({ scope, tabId, operation: "status", input: {} }); + let before = yield* status(); + while (!before.downloads?.length) { + yield* Effect.yieldNow; + before = yield* status(); + } while (desktopDetaches.listenerCount("attach") === 0) yield* Effect.yieldNow; // The user opens the thread after the attach wait ran out, so the desktop mounts its page now. desktopTabs.add(tabId); @@ -1591,6 +1613,10 @@ it.live("an agent tab that went headless moves onto the desktop page that mounts expect(desktopPage.goto.mock.calls.map(([url]) => url)).toContain( "http://localhost:5173/next", ); + // The session stayed open, so the agent's download is still there. + const after = yield* status(); + expect(after.downloads).toEqual(before.downloads); + expect(yield* fs.exists(after.downloads?.[0]?.path ?? "")).toBe(true); }), ).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 5807c8b67c7d0..21ce307b091ee 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -700,7 +700,9 @@ const make = Effect.gen(function* () { else void tab.page.close().catch(constVoid); if (tab.isolatedContext) void tab.page.context().close().catch(constVoid); void tab.recording?.encoder.close().catch(constVoid); - void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid); + // A handed-off tab keeps its downloads; the folder is keyed by tab, not page. + if (!handOff) + void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid); reportLiveTabs(); if (closeSession) { runFork(manager.close({ threadId: tab.threadId, tabId: tab.tabId }).pipe(Effect.ignore)); @@ -2425,7 +2427,13 @@ const make = Effect.gen(function* () { const snapshot = sessions.find( (session) => session.tabId === key.tabId && session.runtime === "server", ); - if (snapshot) yield* Effect.promise(() => ensureTab(snapshot).catch(constVoid)); + const tab = snapshot + ? yield* Effect.promise(() => ensureTab(snapshot).catch(() => null)) + : null; + if (!existing) return; + if (tab) tab.downloads.unshift(...existing.downloads); + else + void NodeFSP.rm(downloadDir(existing), { recursive: true, force: true }).catch(constVoid); }).pipe(Effect.ignore), ), Effect.forkScoped, From f942f5b4cca2a7e04e90aba5e90476dd8b4bba2f Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:42:32 -0700 Subject: [PATCH 3/7] fix(server): stalled browser viewers still learn to reconnect Co-Authored-By: Claude Opus 5.5 --- apps/server/src/preview/ServerBrowser.test.ts | 7 ++++++- apps/server/src/preview/ServerBrowser.ts | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 973929c9bccb7..8fff8c3865170 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -1572,7 +1572,7 @@ it.live("a desktop page that comes back reconnects without waiting for a viewer" it.live("an agent tab that went headless moves onto the desktop page that mounts late", () => Effect.scoped( Effect.gen(function* () { - const { broker, tabId } = yield* ready; + const { browser, broker, tabId } = yield* ready; const headless = contexts[0]!.page; // The agent downloaded a file on the headless page; Chromium writes it, here the test does. const saved = Promise.withResolvers(); @@ -1596,12 +1596,17 @@ it.live("an agent tab that went headless moves onto the desktop page that mounts yield* Effect.yieldNow; before = yield* status(); } + // A remote viewer streams the headless page and has stopped reading. + const viewer = yield* browser.attachViewer(viewerInput(tabId, false)); + stallViewer(viewer, { _tag: "viewport", width: 1, height: 1 }); while (desktopDetaches.listenerCount("attach") === 0) yield* Effect.yieldNow; // The user opens the thread after the attach wait ran out, so the desktop mounts its page now. desktopTabs.add(tabId); desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId }); while (desktopConnections.length === 0) yield* Effect.yieldNow; expect(headless.isClosed()).toBe(true); + // Even with a full queue, the viewer learns to reconnect to the new page. + expect(yield* Queue.take(viewer.output)).toEqual({ _tag: "reconnect" }); yield* broker.invoke({ scope, tabId, diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 21ce307b091ee..388672d12d0a8 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -2237,7 +2237,12 @@ const make = Effect.gen(function* () { if (next._tag === "frame") runFork(next.ack); // State a stalled viewer cannot miss replaces its backlog. It runs // synchronously so an older replacement can never land after a newer one. - else if (next._tag === "gone" || next._tag === "control" || next._tag === "fileChooser") { + else if ( + next._tag === "gone" || + next._tag === "reconnect" || + next._tag === "control" || + next._tag === "fileChooser" + ) { const dropped = Effect.runSyncExit(Queue.clear(output)); if (dropped._tag === "Failure") return; Queue.offerUnsafe(output, next); From c174223965fa7dcdac1feadc8bd85f87a50cab2b Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:47:36 -0700 Subject: [PATCH 4/7] fix(server): a hidden desktop tab no longer freezes the agent's browser tools A snapshot's screenshot waits for the page's next frame. A desktop tab whose thread is not open draws none, so the snapshot held the tab and every later action, even evaluate, waited until the user opened the thread. The capture now gives up after 5 s with an error that says why and what to do. Co-Authored-By: Claude Opus 5.5 --- .../src/preview/ServerBrowserPage.test.ts | 37 ++++++++++++++++++- apps/server/src/preview/ServerBrowserPage.ts | 27 +++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 9a4775bb47173..0a07102dd1829 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -5,7 +5,16 @@ import { type CDPSession, type Page, } from "playwright-core"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vite-plus/test"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vite-plus/test"; import { presentAsChrome } from "./ServerBrowserContexts.ts"; import * as ServerBrowserPage from "./ServerBrowserPage.ts"; @@ -57,6 +66,32 @@ describe("server browser element refs", () => { }; const repeatedRows = `
    ${Array.from({ length: 5 }, (_, i) => `
  • row ${i + 1}
    • `).join("")}
`; + it("gives up on a screenshot when the page draws no frame", async () => { + // A desktop tab whose thread is not open never answers a capture. + const stuck = { + send: (method: string, params?: object) => + method === "Page.captureScreenshot" + ? new Promise(() => {}) + : cdp.send(method as "Page.getLayoutMetrics", params), + } as unknown as CDPSession; + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const snapshot = ServerBrowserPage.snapshot({ + page, + cdp: stuck, + renderScale: 1, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + }); + const rejected = expect(snapshot).rejects.toThrow("did not draw a frame"); + await vi.advanceTimersByTimeAsync(5_000); + await rejected; + } finally { + vi.useRealTimers(); + } + }); + it("clicks the fifth repeated delete control without touching row one", async () => { await page.setContent(repeatedRows); const result = await takeSnapshot(); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 3df4901f469c6..e31edf6dfc2fd 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -175,6 +175,31 @@ export const captureViewport = async ( return data; }; +const SNAPSHOT_CAPTURE_TIMEOUT_MS = 5_000; + +/** + * A capture waits for the page's next frame. A desktop tab whose thread is not + * open draws none, so without a limit the snapshot holds the tab and every + * later action waits behind it. + */ +const captureWithinLimit = (capture: Promise) => { + void capture.catch(constVoid); + let timer: ReturnType | undefined; + const limit = new Promise((_resolve, reject) => { + timer = setTimeout( + () => + reject( + new ServerBrowserOperationError( + "PreviewAutomationExecutionError", + "The page did not draw a frame, so no screenshot was taken. In the desktop app, a tab only draws while the user has its thread open. Use preview_evaluate to read the page, or ask the user to open the thread.", + ), + ), + SNAPSHOT_CAPTURE_TIMEOUT_MS, + ); + }); + return Promise.race([capture, limit]).finally(() => clearTimeout(timer)); +}; + export const snapshot = async (input: { readonly page: Page; readonly cdp: CDPSession; @@ -196,7 +221,7 @@ export const snapshot = async (input: { > >, input.page.ariaSnapshot({ mode: "ai", boxes: true, timeout: DEFAULT_TIMEOUT_MS }), - captureViewport(input.page, input.cdp, { format: "png", scale }), + captureWithinLimit(captureViewport(input.page, input.cdp, { format: "png", scale })), ]); if (state.generation !== generation) { throw new ServerBrowserOperationError( From 93752f567612810e5bfa47b2c6d2b08a3b15daf5 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:51:22 -0700 Subject: [PATCH 5/7] fix(server): every page capture gives up when the page draws no frame Co-Authored-By: Claude Opus 5.5 --- .../src/preview/ServerBrowserPage.test.ts | 23 +++++++---- apps/server/src/preview/ServerBrowserPage.ts | 41 +++++++++++-------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 0a07102dd1829..e397a4a40556d 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -68,15 +68,17 @@ describe("server browser element refs", () => { it("gives up on a screenshot when the page draws no frame", async () => { // A desktop tab whose thread is not open never answers a capture. + let captures = 0; const stuck = { - send: (method: string, params?: object) => - method === "Page.captureScreenshot" - ? new Promise(() => {}) - : cdp.send(method as "Page.getLayoutMetrics", params), + send: (method: string, params?: object) => { + if (method !== "Page.captureScreenshot") + return cdp.send(method as "Page.getLayoutMetrics", params); + captures += 1; + return new Promise(() => {}); + }, } as unknown as CDPSession; - vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); - try { - const snapshot = ServerBrowserPage.snapshot({ + const snapshotOf = () => + ServerBrowserPage.snapshot({ page, cdp: stuck, renderScale: 1, @@ -84,9 +86,14 @@ describe("server browser element refs", () => { networkEntries: [], actionTimeline: [], }); - const rejected = expect(snapshot).rejects.toThrow("did not draw a frame"); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const rejected = expect(snapshotOf()).rejects.toThrow("did not draw a frame"); await vi.advanceTimersByTimeAsync(5_000); await rejected; + // Chromium still holds the first capture, so a retry fails without starting another. + await expect(snapshotOf()).rejects.toThrow("did not draw a frame"); + expect(captures).toBe(1); } finally { vi.useRealTimers(); } diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index e31edf6dfc2fd..7ace8ecb79485 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -150,8 +150,13 @@ const SNAPSHOT_SCRIPT = `(() => { }; })()`; -// Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets. -export const captureViewport = async ( +const CAPTURE_TIMEOUT_MS = 5_000; +const NO_FRAME_MESSAGE = + "The page did not draw a frame, so no screenshot was taken. In the desktop app, a tab only draws while the user has its thread open. Use preview_evaluate to read the page, or ask the user to open the thread."; +/** Captures Chromium has not answered yet; a page only draws one at a time. */ +const pendingCaptures = new WeakSet(); + +const captureFrame = async ( page: Page, cdp: CDPSession, options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, @@ -175,27 +180,27 @@ export const captureViewport = async ( return data; }; -const SNAPSHOT_CAPTURE_TIMEOUT_MS = 5_000; - /** + * Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets. * A capture waits for the page's next frame. A desktop tab whose thread is not - * open draws none, so without a limit the snapshot holds the tab and every - * later action waits behind it. + * open draws none, so the capture gives up after a limit instead of holding + * the tab. A new capture is refused until Chromium answers the old one, which + * still owns the page's capture viewport. */ -const captureWithinLimit = (capture: Promise) => { +export const captureViewport = ( + page: Page, + cdp: CDPSession, + options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, +): Promise => { + const noFrame = () => + new ServerBrowserOperationError("PreviewAutomationExecutionError", NO_FRAME_MESSAGE); + if (pendingCaptures.has(page)) return Promise.reject(noFrame()); + pendingCaptures.add(page); + const capture = captureFrame(page, cdp, options).finally(() => pendingCaptures.delete(page)); void capture.catch(constVoid); let timer: ReturnType | undefined; const limit = new Promise((_resolve, reject) => { - timer = setTimeout( - () => - reject( - new ServerBrowserOperationError( - "PreviewAutomationExecutionError", - "The page did not draw a frame, so no screenshot was taken. In the desktop app, a tab only draws while the user has its thread open. Use preview_evaluate to read the page, or ask the user to open the thread.", - ), - ), - SNAPSHOT_CAPTURE_TIMEOUT_MS, - ); + timer = setTimeout(() => reject(noFrame()), CAPTURE_TIMEOUT_MS); }); return Promise.race([capture, limit]).finally(() => clearTimeout(timer)); }; @@ -221,7 +226,7 @@ export const snapshot = async (input: { > >, input.page.ariaSnapshot({ mode: "ai", boxes: true, timeout: DEFAULT_TIMEOUT_MS }), - captureWithinLimit(captureViewport(input.page, input.cdp, { format: "png", scale })), + captureViewport(input.page, input.cdp, { format: "png", scale }), ]); if (state.generation !== generation) { throw new ServerBrowserOperationError( From 3dd0bd1c3cc2b510cab9dd9824065046eb4f7240 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 01:58:41 -0700 Subject: [PATCH 6/7] fix(desktop): a hidden browser tab draws while the agent captures it, until the turn ends When the server asks for a screenshot of a desktop tab, the desktop now tells the web app, which keeps that webview drawing (the same lease recording uses) until the thread's turn ends. Before, a tab in a thread the user was not viewing sat offscreen and drew nothing, so agent screenshots never finished. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/ipc/channels.ts | 1 + apps/desktop/src/ipc/methods/preview.ts | 3 ++ apps/desktop/src/preload.ts | 8 +++ .../src/preview/DesktopBrowserHost.test.ts | 21 ++++++++ .../desktop/src/preview/DesktopBrowserHost.ts | 8 +++ apps/desktop/src/preview/Manager.ts | 50 +++++++++++++++---- apps/web/src/browser/ElectronBrowserHost.tsx | 9 ++++ .../src/browser/agentFrameRequests.test.ts | 42 ++++++++++++++++ apps/web/src/browser/agentFrameRequests.ts | 30 +++++++++++ packages/contracts/src/ipc.ts | 2 + 10 files changed, 164 insertions(+), 10 deletions(-) create mode 100644 apps/web/src/browser/agentFrameRequests.test.ts create mode 100644 apps/web/src/browser/agentFrameRequests.ts diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index a6b617385b578..a8f07bac29b1e 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -111,6 +111,7 @@ export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save"; export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame"; export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change"; export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event"; +export const PREVIEW_FRAME_REQUEST_CHANNEL = "desktop:preview-frame-request"; export const PREVIEW_OPEN_LINK_CHANNEL = "desktop:preview-open-link"; export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 94c56464f7c47..980cffdc855c3 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -54,6 +54,9 @@ export const installPreviewEventForwarding = Effect.fn( yield* manager.subscribePointerEvents((event) => electronWindow.sendAll(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, event), ); + yield* manager.subscribeFrameRequests((tabId) => + electronWindow.sendAll(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, tabId), + ); yield* manager.subscribeOpenLinks((event) => electronWindow.sendAll(IpcChannels.PREVIEW_OPEN_LINK_CHANNEL, event), ); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index efd49ffa38648..90136c5703a12 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -429,6 +429,14 @@ contextBridge.exposeInMainWorld("desktopBridge", { return () => ipcRenderer.removeListener(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, wrappedListener); }, + onFrameRequest: (listener) => { + const wrappedListener = (_event: Electron.IpcRendererEvent, tabId: unknown) => { + if (typeof tabId === "string") listener(tabId); + }; + ipcRenderer.on(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener); + return () => + ipcRenderer.removeListener(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener); + }, onOpenLink: (listener) => { const wrappedListener = (_event: Electron.IpcRendererEvent, linkEvent: unknown) => { if (typeof linkEvent !== "object" || linkEvent === null) return; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 81ea040701f0e..c1d3a6510e30f 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -148,4 +148,25 @@ describe("DesktopBrowserHost", () => { expect(host.humanStartedDownload(debuggee.tab.webContents)).toBe(false); }), ); + + it.effect("asks the tab to draw when the server waits for a capture", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + host.attach(key, makeDebuggee().tab); + const requests = yield* host.frameRequests.pipe( + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Effect.yieldNow; + const command = (id: number, method: string) => + host.handleCommandLine( + encodeJson({ type: "cdp", ...key, message: encodeJson({ id, method, params: {} }) }), + ); + // Reading the page needs no frame; a capture waits for the next one. + yield* command(1, "DOM.getDocument"); + yield* command(2, "Page.captureScreenshot"); + expect(yield* Fiber.join(requests)).toEqual([key]); + }).pipe(Effect.scoped), + ); }); diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index cfb1ff753c470..1d8cfb902879b 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -31,6 +31,8 @@ const lineEncoder = new TextEncoder(); /** CDP commands through which an agent clicks, types, or navigates the page. */ const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; +/** A capture waits for the page's next frame, which a hidden tab only draws on request. */ +const CAPTURE_COMMAND = /"method":"Page\.captureScreenshot"/; /** How long after an agent's input a download still counts as the agent's. */ const AGENT_DOWNLOAD_WINDOW_MS = 5_000; @@ -93,6 +95,8 @@ export class DesktopBrowserHost extends Context.Service< readonly x: number; readonly y: number; }>; + /** Server tabs whose page the server is waiting to capture, so they must draw. */ + readonly frameRequests: Stream.Stream; } >()("@t3tools/desktop/preview/DesktopBrowserHost") {} @@ -104,6 +108,7 @@ export const make = Effect.gen(function* () { readonly x: number; readonly y: number; }>(16); + const frameRequests = yield* PubSub.sliding(16); const runFork = Effect.runForkWith(yield* Effect.context()); const tabs = new Map(); const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event)); @@ -204,6 +209,8 @@ export const make = Effect.gen(function* () { return; } if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now(); + if (CAPTURE_COMMAND.test(command.value.message)) + runFork(PubSub.publish(frameRequests, tab.key)); relayFor(tab).receive(command.value.message); }); @@ -226,6 +233,7 @@ export const make = Effect.gen(function* () { return DesktopBrowserHost.of({ pointers: Stream.fromPubSub(pointers), + frameRequests: Stream.fromPubSub(frameRequests), // Subscribes before announcing, so no attach falls between the two. events: Stream.unwrap( Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index aa36055452eb8..ddf1c437238e0 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -415,6 +415,8 @@ type RecordingInputListener = (event: DesktopPreviewRecordingInputEvent) => Effe type PointerEventListener = (event: DesktopPreviewPointerEvent) => Effect.Effect; +type FrameRequestListener = (tabId: string) => Effect.Effect; + type OpenLinkListener = (event: DesktopPreviewOpenLinkEvent) => Effect.Effect; /** @@ -562,6 +564,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const attachedRef = yield* Ref.make>(new Map()); const listenersRef = yield* Ref.make>(new Set()); const pointerEventListenersRef = yield* Ref.make>(new Set()); + const frameRequestListenersRef = yield* Ref.make>(new Set()); const openLinkListenersRef = yield* Ref.make>(new Set()); const recordingInputListenersRef = yield* Ref.make>( new Set(), @@ -838,6 +841,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function | "recording-frame" | "recording-input" | "pointer-event" + | "frame-request" | "open-link", tabId: string, delivery: () => Effect.Effect, @@ -2411,7 +2415,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return; } if (afterAttach.serverTab) { - yield* listenForAgentPointers; + yield* listenForAgentSignals; browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger }); } if (afterAttach.colorScheme !== "system") { @@ -3343,17 +3347,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { discard: true }, ); }); - // Listening starts with the first tab of the desktop's own server. - let pointersStarted = false; - const listenForAgentPointers = Effect.suspend(() => { - if (pointersStarted) return Effect.void; - pointersStarted = true; - return browserHost.pointers.pipe( - Stream.runForEach(emitAgentPointer), - Effect.forkIn(parentScope), - Effect.asVoid, + /** Tells the renderer that the server waits for a frame of this tab, so it must draw. */ + const emitFrameRequest = Effect.fn("PreviewManager.emitFrameRequest")(function* (key: { + readonly threadId: string; + readonly tabId: string; + }) { + const tab = [...(yield* SynchronizedRef.get(tabsRef)).values()].find( + (candidate) => + candidate.serverTab?.threadId === key.threadId && candidate.serverTab.tabId === key.tabId, + ); + if (!tab) return; + const listeners = yield* Ref.get(frameRequestListenersRef); + yield* Effect.forEach( + listeners, + (listener) => deliverEvent("frame-request", tab.tabId, () => listener(tab.tabId)), + { discard: true }, ); }); + // Listening starts with the first tab of the desktop's own server. + let agentSignalsStarted = false; + const listenForAgentSignals = Effect.suspend(() => { + if (agentSignalsStarted) return Effect.void; + agentSignalsStarted = true; + return Effect.all([ + browserHost.pointers.pipe(Stream.runForEach(emitAgentPointer), Effect.forkIn(parentScope)), + browserHost.frameRequests.pipe( + Stream.runForEach(emitFrameRequest), + Effect.forkIn(parentScope), + ), + ]).pipe(Effect.asVoid); + }); const emitOpenLink = Effect.fn("PreviewManager.emitOpenLink")(function* ( event: DesktopPreviewOpenLinkEvent, @@ -3386,6 +3409,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function [ Ref.set(listenersRef, new Set()), Ref.set(pointerEventListenersRef, new Set()), + Ref.set(frameRequestListenersRef, new Set()), Ref.set(openLinkListenersRef, new Set()), Ref.set(recordingFrameListenersRef, new Set()), Ref.set(recordingInputListenersRef, new Set()), @@ -3432,6 +3456,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function stopRecording, subscribePointerEvents: (listener: PointerEventListener) => subscribe(pointerEventListenersRef, listener), + subscribeFrameRequests: (listener: FrameRequestListener) => + subscribe(frameRequestListenersRef, listener), subscribeOpenLinks: (listener: OpenLinkListener) => subscribe(openLinkListenersRef, listener), subscribeRecordingInputs: (listener: RecordingInputListener) => subscribe(recordingInputListenersRef, listener), @@ -3674,6 +3700,9 @@ export class PreviewManager extends Context.Service< readonly subscribePointerEvents: ( listener: PointerEventListener, ) => Effect.Effect; + readonly subscribeFrameRequests: ( + listener: FrameRequestListener, + ) => Effect.Effect; readonly subscribeOpenLinks: ( listener: OpenLinkListener, ) => Effect.Effect; @@ -3819,6 +3848,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { saveRecording: operations.saveRecording, subscribeStateChanges: operations.subscribeStateChanges, subscribePointerEvents: operations.subscribePointerEvents, + subscribeFrameRequests: operations.subscribeFrameRequests, subscribeOpenLinks: operations.subscribeOpenLinks, subscribeRecordingFrames: operations.subscribeRecordingFrames, subscribeRecordingInputs: operations.subscribeRecordingInputs, diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index b076a30a5221f..2e602201d6ec9 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -15,6 +15,7 @@ import { previewEnvironment } from "~/state/preview"; import { useEnvironmentScope } from "~/state/session"; import { useAtomCommand } from "~/state/use-atom-command"; +import { holdFramesForTurn } from "./agentFrameRequests"; import { readPreviewAnnotationTheme } from "./annotationTheme"; import { useBrowserDefaults } from "./browserDefaults"; import { useBrowserPointerStore } from "./browserPointerStore"; @@ -106,6 +107,14 @@ export function ElectronBrowserHost() { sessions.map((session) => [session.runtimeTabId, session]), ); }, [sessions]); + useEffect(() => { + const preview = window.desktopBridge?.preview; + if (!preview) return; + return preview.onFrameRequest((tabId) => { + const source = sessionByRuntimeTabId.current.get(tabId); + if (source) holdFramesForTurn(tabId, source.threadRef); + }); + }, []); useEffect(() => { const preview = window.desktopBridge?.preview; if (!preview) return; diff --git a/apps/web/src/browser/agentFrameRequests.test.ts b/apps/web/src/browser/agentFrameRequests.test.ts new file mode 100644 index 0000000000000..e7f161fcd3217 --- /dev/null +++ b/apps/web/src/browser/agentFrameRequests.test.ts @@ -0,0 +1,42 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { Atom } from "effect/reactivity"; +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { appAtomRegistry } from "~/rpc/atomRegistry"; + +import { holdFramesForTurn } from "./agentFrameRequests"; +import { useBrowserSurfaceStore } from "./browserSurfaceStore"; + +const shell = vi.hoisted(() => ({ atom: null as unknown })); +vi.mock("~/state/threads", () => ({ + environmentThreadShells: { threadShellAtom: () => shell.atom }, +})); + +type Shell = { readonly runtime: { readonly status: string } | null }; +const threadRef = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), +}; +const shellAtom = Atom.make(null).pipe(Atom.keepAlive); +shell.atom = shellAtom; +const setStatus = (status: string) => appAtomRegistry.set(shellAtom, { runtime: { status } }); +const drawing = (tabId: string) => useBrowserSurfaceStore.getState().activityByTabId[tabId] ?? 0; + +describe("holdFramesForTurn", () => { + beforeEach(() => setStatus("idle")); + + it("keeps a hidden tab drawing until the thread's turn ends", () => { + setStatus("running"); + holdFramesForTurn("tab-1", threadRef); + // Later captures in the same turn reuse the lease. + holdFramesForTurn("tab-1", threadRef); + expect(drawing("tab-1")).toBe(1); + setStatus("idle"); + expect(drawing("tab-1")).toBe(0); + }); + + it("does not draw for a capture outside a turn", () => { + holdFramesForTurn("tab-2", threadRef); + expect(drawing("tab-2")).toBe(0); + }); +}); diff --git a/apps/web/src/browser/agentFrameRequests.ts b/apps/web/src/browser/agentFrameRequests.ts new file mode 100644 index 0000000000000..6fc060e6c5309 --- /dev/null +++ b/apps/web/src/browser/agentFrameRequests.ts @@ -0,0 +1,30 @@ +import { threadRuntimeIsActive } from "@t3tools/client-runtime/state/models"; +import type { ScopedThreadRef } from "@t3tools/contracts"; + +import { appAtomRegistry } from "~/rpc/atomRegistry"; +import { environmentThreadShells } from "~/state/threads"; + +import { acquireBrowserSurfaceActivity } from "./browserSurfaceStore"; + +const leases = new Map void>(); + +/** + * Keeps a hidden tab drawing after the server asked for one of its frames, until + * the thread's turn ends. A tab parked offscreen draws nothing, so an agent's + * screenshot would wait forever. Later requests in the same turn reuse the lease. + */ +export function holdFramesForTurn(tabId: string, threadRef: ScopedThreadRef): void { + if (leases.has(tabId)) return; + const shellAtom = environmentThreadShells.threadShellAtom(threadRef); + if (!threadRuntimeIsActive(appAtomRegistry.get(shellAtom)?.runtime)) return; + const releaseActivity = acquireBrowserSurfaceActivity(tabId); + const release = () => { + leases.delete(tabId); + unsubscribe(); + releaseActivity(); + }; + const unsubscribe = appAtomRegistry.subscribe(shellAtom, (shell) => { + if (!threadRuntimeIsActive(shell?.runtime)) release(); + }); + leases.set(tabId, release); +} diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index e521c4c37271c..7b86c3131365c 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1352,6 +1352,8 @@ export interface DesktopPreviewBridge { }; onStateChange: (listener: (tabId: string, state: DesktopPreviewTabState) => void) => () => void; onPointerEvent: (listener: (event: DesktopPreviewPointerEvent) => void) => () => void; + /** The server waits for a frame of this hidden tab, so it should draw until the turn ends. */ + onFrameRequest: (listener: (tabId: string) => void) => () => void; onOpenLink: (listener: (event: DesktopPreviewOpenLinkEvent) => void) => () => void; } From b9fa58f765e44c374c2bfbffaa07f9d0f240b102 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 12 Oct 2026 18:14:38 -0700 Subject: [PATCH 7/7] fix(preview): keep headless pages and stream them across thread switches Keep the existing page, session state, downloads, and recordings when desktop rendering arrives late. Use native frame leases for background captures, including threads without a loaded shell. Co-authored-by: davidesala Co-authored-by: GPT-6.1 Sol --- apps/server/src/preview/Manager.test.ts | 83 ++++ apps/server/src/preview/Manager.ts | 50 +- apps/server/src/preview/ServerBrowser.test.ts | 442 ++++++++++++++++-- apps/server/src/preview/ServerBrowser.ts | 82 ++-- apps/server/src/preview/ServerBrowserPage.ts | 2 +- apps/web/src/browser/ElectronBrowserHost.tsx | 20 +- .../src/browser/HostedBrowserWebview.test.tsx | 45 ++ .../src/browser/agentFrameRequests.test.ts | 75 ++- apps/web/src/browser/agentFrameRequests.ts | 47 +- apps/web/src/browser/previewRuntime.test.ts | 53 ++- apps/web/src/browser/previewRuntime.ts | 8 +- .../components/preview/PreviewView.test.tsx | 131 +++++- .../src/components/preview/PreviewView.tsx | 3 +- .../preview/ThreadPreviewMiniPlayer.tsx | 4 +- .../preview/previewFallback.test.tsx | 227 +++++++++ .../preview/usePreviewBridge.test.ts | 115 ++++- .../components/preview/usePreviewBridge.ts | 19 +- packages/contracts/src/ipc.ts | 2 +- packages/contracts/src/preview.test.ts | 75 ++- packages/contracts/src/preview.ts | 10 +- packages/contracts/src/previewAutomation.ts | 3 + 21 files changed, 1349 insertions(+), 147 deletions(-) create mode 100644 apps/web/src/components/preview/previewFallback.test.tsx diff --git a/apps/server/src/preview/Manager.test.ts b/apps/server/src/preview/Manager.test.ts index 48657d85d9023..96587a128e566 100644 --- a/apps/server/src/preview/Manager.test.ts +++ b/apps/server/src/preview/Manager.test.ts @@ -100,6 +100,89 @@ it.layer(layer)("PreviewManager", (it) => { }), ); + it.effect("publishes the server's page mode once and preserves a headless choice", () => + Effect.gen(function* () { + const threadId = freshThreadId(); + const manager = yield* PreviewManager.PreviewManager; + const collector = yield* collectEvents; + const opened = yield* manager.open({ + threadId, + runtime: "server", + url: "http://localhost:5173/", + profileId: "work", + automationOwner: "agent", + }); + const native = yield* manager.setServerPageMode({ + threadId, + tabId: opened.tabId, + mode: "native", + }); + const headless = yield* manager.setServerPageMode({ + threadId, + tabId: opened.tabId, + mode: "headless", + }); + expect(native.serverPageMode).toBe("native"); + expect(headless).toMatchObject({ + ...opened, + serverPageMode: "headless", + updatedAt: expect.any(String), + }); + const before = yield* manager.list({ threadId }); + for (const mode of ["headless", "native"] as const) { + expect(yield* manager.setServerPageMode({ threadId, tabId: opened.tabId, mode })).toEqual( + headless, + ); + } + const after = yield* manager.list({ threadId }); + expect(after).toEqual(before); + const events = yield* collector.drain; + expect(events.map((event) => event.type)).toEqual(["opened", "resized", "resized"]); + expect(events.at(-1)).toMatchObject({ snapshot: headless, revision: after.revision }); + yield* manager.reportStatus({ + threadId, + tabId: opened.tabId, + serverControlled: true, + navStatus: { _tag: "Success", url: "http://localhost:5173/next", title: "Next" }, + canGoBack: true, + canGoForward: false, + }); + yield* manager.resize({ threadId, tabId: opened.tabId, viewport: { _tag: "fill" } }); + expect((yield* manager.list({ threadId })).sessions[0]?.serverPageMode).toBe("headless"); + }), + ); + + it.effect( + "allows only server sessions to select a page mode, and never recreates a closed tab", + () => + Effect.gen(function* () { + const threadId = freshThreadId(); + const manager = yield* PreviewManager.PreviewManager; + const desktop = yield* manager.open({ threadId, serverPageMode: "headless" }); + expect(desktop).not.toHaveProperty("serverPageMode"); + expect( + (yield* manager + .setServerPageMode({ threadId, tabId: desktop.tabId, mode: "headless" }) + .pipe(Effect.flip))._tag, + ).toBe("PreviewControlRequiredError"); + const server = yield* manager.open({ + threadId, + runtime: "server", + serverPageMode: "headless", + }); + expect(server.serverPageMode).toBe("headless"); + yield* manager.close({ threadId, tabId: server.tabId }); + expect( + (yield* manager + .setServerPageMode({ threadId, tabId: server.tabId, mode: "headless" }) + .pipe(Effect.flip))._tag, + ).toBe("PreviewSessionLookupError"); + expect( + (yield* manager.list({ threadId })).sessions.map((session) => session.tabId), + ).toEqual([desktop.tabId]); + }), + ); + it.effect("lets any client size, theme, and zoom a server tab", () => Effect.gen(function* () { const threadId = freshThreadId(); diff --git a/apps/server/src/preview/Manager.ts b/apps/server/src/preview/Manager.ts index 9420e07a5a3a7..a3d79f5502020 100644 --- a/apps/server/src/preview/Manager.ts +++ b/apps/server/src/preview/Manager.ts @@ -48,6 +48,8 @@ export class PreviewManager extends Context.Service< readonly open: ( input: PreviewOpenInput & { readonly automationOwner?: string; + /** Only the server browser selects how its page is presented. */ + readonly serverPageMode?: PreviewSessionSnapshot["serverPageMode"]; /** Runs before the `opened` event publishes, so subscribers find state keyed by the tab. */ readonly beforePublish?: (snapshot: PreviewSessionSnapshot) => void; }, @@ -58,6 +60,13 @@ export class PreviewManager extends Context.Service< readonly reportStatus: ( input: PreviewReportStatusInput & { readonly serverControlled?: boolean }, ) => Effect.Effect; + /** Publishes the backing page selected by the server; headless cannot be promoted later. */ + readonly setServerPageMode: ( + input: PreviewCloseInput & { + readonly tabId: string; + readonly mode: "native" | "headless"; + }, + ) => Effect.Effect; readonly requestReveal: ( input: PreviewCloseInput & { readonly tabId: string; readonly force: boolean }, ) => Effect.Effect; @@ -221,6 +230,9 @@ export const make = Effect.gen(function* PreviewManagerMake() { viewport: input.viewport ?? FILL_PREVIEW_VIEWPORT, ...(input.profileId === undefined ? {} : { profileId: input.profileId }), ...(runtime === undefined ? {} : { runtime }), + ...(runtime === "server" && input.serverPageMode !== undefined + ? { serverPageMode: input.serverPageMode } + : {}), ...(runtime === "server" && input.automationOwner !== undefined ? { automationOwner: input.automationOwner } : {}), @@ -337,6 +349,40 @@ export const make = Effect.gen(function* PreviewManagerMake() { ); }); + const setServerPageMode: PreviewManager["Service"]["setServerPageMode"] = Effect.fn( + "PreviewManager.setServerPageMode", + )(function* (input) { + return yield* mutateExistingSession( + input.threadId, + input.tabId, + Effect.fn("PreviewManager.setSessionPageMode")(function* (session) { + if (session.snapshot.runtime !== "server") + return yield* new PreviewControlRequiredError({ tabId: input.tabId }); + if ( + session.snapshot.serverPageMode === input.mode || + session.snapshot.serverPageMode === "headless" + ) + return { next: session, emit: null, result: session.snapshot }; + const snapshot: PreviewSessionSnapshot = { + ...session.snapshot, + serverPageMode: input.mode, + updatedAt: yield* currentIsoTimestamp, + }; + return { + next: { ...session, snapshot }, + emit: { + type: "resized", + threadId: session.threadId, + tabId: session.tabId, + createdAt: snapshot.updatedAt, + snapshot, + }, + result: snapshot, + }; + }), + ); + }); + const resize: PreviewManager["Service"]["resize"] = Effect.fn("PreviewManager.resize")( function* (input) { return yield* mutateExistingSession( @@ -456,7 +502,8 @@ export const make = Effect.gen(function* PreviewManagerMake() { const list: PreviewManager["Service"]["list"] = Effect.fn("PreviewManager.list")( function* (input) { - return yield* SynchronizedRef.get(stateRef).pipe( + // Events publish inside a mutation; their consumers must wait for its state commit. + return yield* SynchronizedRef.modify(stateRef, (state) => [state, state] as const).pipe( Effect.map((state): PreviewListResult => ({ sessions: sessionsForThread(state, input.threadId) .map((s) => s.snapshot) @@ -504,6 +551,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { requestReveal, navigate, reportStatus, + setServerPageMode, resize, adjust, refresh, diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 8fff8c3865170..b92f35887bd98 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -40,6 +40,7 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ } async contextFor(profileId: string, isolationKey?: string) { contextRequests.push({ profileId, isolated: isolationKey !== undefined }); + contextRequested?.resolve(); if (contextFailure) throw contextFailure; await contextGate?.promise; const context = makeContext(this.onClose); @@ -47,7 +48,9 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ return context as unknown as BrowserContext; } async scratchPage() { - return makeContext().page as unknown as Page; + const page = makeContext().page; + scratchPages.push(page); + return page as unknown as Page; } async connectDesktopPage(endpoint: string) { const context = makeContext(); @@ -106,7 +109,7 @@ function makeContext(onClose?: (context: BrowserContext) => void) { reload: vi.fn(async () => {}), emulateMedia: vi.fn(async () => {}), waitForLoadState: vi.fn(async () => {}), - evaluate: vi.fn(async () => ({ + evaluate: vi.fn(async (): Promise => ({ url, title: "test page", loading: false, @@ -150,14 +153,21 @@ function makeContext(onClose?: (context: BrowserContext) => void) { } const contexts: ReturnType[] = []; +const scratchPages: ReturnType["page"][] = []; /** The profile and isolation each headless tab asked its context for. */ const contextRequests: Array<{ profileId: string; isolated: boolean }> = []; let contextGate: PromiseWithResolvers | null = null; +let contextRequested: PromiseWithResolvers | null = null; type ClipboardBinding = (source: { page: unknown }, text: unknown) => void; let clipboardBinding: ClipboardBinding | null = null; let contextFailure: Error | null = null; /** Server tabs the fake desktop renders, and the endpoints the server connected to. */ let desktopRendersNext = false; +let desktopAvailable = false; +let desktopAttachGate: PromiseWithResolvers | null = null; +let desktopAttachStarted: PromiseWithResolvers | null = null; +let desktopDetachConsumed: PromiseWithResolvers | null = null; +const desktopAttachRequests: string[] = []; /** Pages the fake desktop takes back or returns; the channel's streams emit them. */ const desktopDetaches = new NodeEvents.EventEmitter(); const desktopTabs = new Set(); @@ -169,7 +179,10 @@ const desktopRenders = (tabId: string) => { return desktopTabs.has(tabId); }; const releasedDesktopTabs: Array = []; -const desktopConnections: Array<{ endpoint: string; context: ReturnType }> = []; +const desktopConnections: Array<{ + endpoint: string; + context: ReturnType; +}> = []; const testThread = { threadId: ThreadId.make("browser-test-thread"), providerSessionId: "agent-a", @@ -200,9 +213,16 @@ const dependencies = Layer.mergeAll( installed: Effect.die("mock Chromium does not need an executable"), }), Layer.succeed(DesktopChannel.DesktopBrowserChannel, { - // Only tabs a test marks render on the desktop; the rest stay headless. - available: true, - awaitAttached: (key) => Effect.sync(() => desktopRenders(key.tabId)), + // Standalone tests have no desktop; native tests explicitly enable it. + get available() { + return desktopAvailable || desktopRendersNext || desktopTabs.size > 0; + }, + awaitAttached: (key) => + Effect.promise(async () => { + desktopAttachRequests.push(key.tabId); + desktopAttachStarted?.resolve(); + return desktopAttachGate ? desktopAttachGate.promise : desktopRenders(key.tabId); + }), detached: Stream.callback<{ threadId: string; tabId: string }>((queue) => Effect.acquireRelease( Effect.sync(() => { @@ -213,7 +233,7 @@ const dependencies = Layer.mergeAll( }), (onDetach) => Effect.sync(() => desktopDetaches.off("detach", onDetach)), ), - ), + ).pipe(Stream.tap(() => Effect.sync(() => desktopDetachConsumed?.resolve()))), attached: Stream.callback<{ threadId: string; tabId: string }>((queue) => Effect.acquireRelease( Effect.sync(() => { @@ -269,11 +289,18 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({ beforeEach(() => { contexts.length = 0; + scratchPages.length = 0; contextRequests.length = 0; contextGate = null; + contextRequested = null; contextFailure = null; desktopTabs.clear(); desktopRendersNext = false; + desktopAvailable = false; + desktopAttachGate = null; + desktopAttachStarted = null; + desktopDetachConsumed = null; + desktopAttachRequests.length = 0; releasedDesktopTabs.length = 0; desktopConnections.length = 0; }); @@ -960,12 +987,14 @@ it.live("a popup from a person's click is shown to that person, with its opener const browser = yield* ServerBrowser.ServerBrowser; const manager = yield* Manager.PreviewManager; yield* Effect.yieldNow; + desktopAvailable = true; const opened = yield* manager.open({ threadId: scope.thread.threadId, runtime: "server" }); const watcher = yield* browser.attachViewer(viewerInput(opened.tabId, false)); const viewer = yield* browser.attachViewer(viewerInput(opened.tabId, true)); yield* viewer.input({ type: "takeControl" }); const opener = contexts[0]!.page; const popup = makeContext(); + const events = yield* manager.subscribeEvents; opener.emit("popup", popup.page); let shown = yield* Queue.take(viewer.output); while (shown._tag !== "popup") shown = yield* Queue.take(viewer.output); @@ -973,6 +1002,16 @@ it.live("a popup from a person's click is shown to that person, with its opener expect(sessions.map((session) => session.tabId)).toEqual( expect.arrayContaining([opened.tabId, shown.tabId]), ); + expect(sessions.find((session) => session.tabId === shown.tabId)?.serverPageMode).toBe( + "headless", + ); + expect( + (yield* PubSub.takeUpTo(events, 10)).find((event) => event.type === "opened"), + ).toMatchObject({ snapshot: { tabId: shown.tabId, serverPageMode: "headless" } }); + yield* browser.attachViewer(viewerInput(shown.tabId, false)); + expect(desktopAttachRequests).toEqual([opened.tabId]); + expect(desktopConnections).toEqual([]); + expect(contexts).toHaveLength(1); expect((yield* Queue.clear(watcher.output)).some((item) => item._tag === "popup")).toBe( false, ); @@ -981,6 +1020,48 @@ it.live("a popup from a person's click is shown to that person, with its opener ).pipe(Effect.provide(layer)), ); +it.live("a popup closed before adoption does not leave its page behind", () => + Effect.scoped( + Effect.gen(function* () { + const { tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const events = yield* manager.subscribeEvents; + const reading = Promise.withResolvers(); + const read = Promise.withResolvers(); + const closed = Promise.withResolvers(); + const list = manager.list; + const spy = vi.spyOn(manager, "list").mockImplementationOnce((input) => + Effect.promise(async () => { + reading.resolve(); + await read.promise; + }).pipe(Effect.andThen(list(input))), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + read.resolve(); + spy.mockRestore(); + }), + ); + const popup = makeContext(); + popup.page.once("close", () => closed.resolve()); + contexts[0]!.page.emit("popup", popup.page); + yield* Effect.promise(() => reading.promise); + const opened = yield* PubSub.take(events); + expect(opened.type).toBe("opened"); + yield* manager.close({ threadId: scope.thread.threadId, tabId: opened.tabId }); + read.resolve(); + yield* Effect.promise(() => closed.promise); + expect(popup.page.isClosed()).toBe(true); + expect(contexts[0]!.page.isClosed()).toBe(false); + expect( + (yield* manager.list({ threadId: scope.thread.threadId })).sessions.map( + (session) => session.tabId, + ), + ).toEqual([tabId]); + }), + ).pipe(Effect.provide(layer)), +); + it.live("closing a tab while a viewer is still opening it does not leave its page behind", () => Effect.scoped( Effect.gen(function* () { @@ -1310,6 +1391,256 @@ it.live("viewers see the agent's pointer move to its target and click there", () ).pipe(Effect.provide(layer)), ); +it.live( + "publishes a headless fallback before launch and keeps the same page after late native events", + () => + Effect.scoped( + Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + const events = yield* manager.subscribeEvents; + yield* Effect.yieldNow; + desktopAvailable = true; + const launching = Promise.withResolvers(); + const launch = Promise.withResolvers(); + contextRequested = launching; + contextGate = launch; + yield* Effect.addFinalizer(() => Effect.sync(() => launch.resolve())); + const opening = yield* broker + .invoke({ + scope, + operation: "open", + input: { url: "http://localhost:5173/fallback", reuseExistingTab: false, show: false }, + }) + .pipe(Effect.forkScoped); + yield* Effect.promise(() => launching.promise); + const listed = yield* manager.list({ threadId: scope.thread.threadId }); + const snapshot = listed.sessions[0]!; + expect(snapshot.serverPageMode).toBe("headless"); + expect(contexts).toEqual([]); + expect( + (yield* PubSub.takeUpTo(events, 10)).map((event) => ({ + type: event.type, + mode: "snapshot" in event ? event.snapshot.serverPageMode : undefined, + })), + ).toEqual([ + { type: "opened", mode: undefined }, + { type: "resized", mode: "headless" }, + ]); + // A native guest arriving while headless is still launching must not become another page. + const queried = Promise.withResolvers(); + const list = manager.list; + const spy = vi + .spyOn(manager, "list") + .mockImplementation((input) => + list(input).pipe(Effect.tap(() => Effect.sync(() => queried.resolve()))), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => spy.mockRestore())); + desktopTabs.add(snapshot.tabId); + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId: snapshot.tabId }); + yield* Effect.promise(() => queried.promise); + launch.resolve(); + const opened = yield* Fiber.join(opening); + expect(opened).toMatchObject({ + available: true, + tabId: snapshot.tabId, + url: "http://localhost:5173/fallback", + }); + expect(contexts).toHaveLength(1); + expect(contexts[0]!.page.goto).toHaveBeenCalledOnce(); + expect(desktopConnections).toEqual([]); + const viewer = yield* browser.attachViewer(viewerInput(snapshot.tabId, false)); + expect((yield* Queue.clear(viewer.output)).some((item) => item._tag === "frame")).toBe( + true, + ); + const consumed = Promise.withResolvers(); + desktopDetachConsumed = consumed; + desktopTabs.delete(snapshot.tabId); + desktopDetaches.emit("detach", { threadId: scope.thread.threadId, tabId: snapshot.tabId }); + yield* Effect.promise(() => consumed.promise); + expect( + yield* broker.invoke({ + scope, + tabId: PreviewTabId.make(snapshot.tabId), + operation: "evaluate", + input: { expression: "42" }, + }), + ).toBe("evaluated"); + expect(contexts[0]!.page.close).not.toHaveBeenCalled(); + expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toHaveLength(1); + }), + ).pipe(Effect.provide(layer)), +); + +it.live( + "a chosen headless page stays headless when its failed launch is retried after native attachment", + () => + Effect.scoped( + Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + desktopAvailable = true; + contextFailure = new Error("launch failed"); + yield* broker + .invoke({ + scope, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }) + .pipe(Effect.flip); + const snapshot = (yield* manager.list({ threadId: scope.thread.threadId })).sessions[0]!; + expect(snapshot.serverPageMode).toBe("headless"); + contextFailure = null; + desktopTabs.add(snapshot.tabId); + const viewer = yield* browser.attachViewer(viewerInput(snapshot.tabId, false)); + expect((yield* Queue.clear(viewer.output)).some((item) => item._tag === "frame")).toBe( + true, + ); + expect(desktopConnections).toEqual([]); + expect(contexts).toHaveLength(1); + expect(desktopAttachRequests).toEqual([snapshot.tabId]); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("closing a tab during its native wait prevents a headless fallback", () => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + desktopAvailable = true; + const waiting = Promise.withResolvers(); + const attached = Promise.withResolvers(); + desktopAttachStarted = waiting; + desktopAttachGate = attached; + yield* Effect.addFinalizer(() => Effect.sync(() => attached.resolve(false))); + const opening = yield* broker + .invoke({ + scope, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Effect.promise(() => waiting.promise); + const snapshot = (yield* manager.list({ threadId: scope.thread.threadId })).sessions[0]!; + yield* manager.close({ threadId: scope.thread.threadId, tabId: snapshot.tabId }); + attached.resolve(false); + yield* Fiber.join(opening); + expect(contextRequests).toEqual([]); + expect(contexts).toEqual([]); + expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toEqual([]); + }), + ).pipe(Effect.provide(layer)), +); + +it.live( + "closing a tab while headless launches discards its page and prevents late native resurrection", + () => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + desktopAvailable = true; + const launching = Promise.withResolvers(); + const launch = Promise.withResolvers(); + contextRequested = launching; + contextGate = launch; + yield* Effect.addFinalizer(() => Effect.sync(() => launch.resolve())); + const opening = yield* broker + .invoke({ + scope, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Effect.promise(() => launching.promise); + const snapshot = (yield* manager.list({ threadId: scope.thread.threadId })).sessions[0]!; + yield* manager.close({ threadId: scope.thread.threadId, tabId: snapshot.tabId }); + launch.resolve(); + yield* Fiber.join(opening); + expect(contexts[0]!.page.isClosed()).toBe(true); + expect(contexts[0]!.close).toHaveBeenCalledOnce(); + expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toEqual([]); + const queried = Promise.withResolvers(); + const list = manager.list; + const spy = vi + .spyOn(manager, "list") + .mockImplementation((input) => + list(input).pipe(Effect.tap(() => Effect.sync(() => queried.resolve()))), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => spy.mockRestore())); + desktopTabs.add(snapshot.tabId); + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId: snapshot.tabId }); + yield* Effect.promise(() => queried.promise); + expect(desktopConnections).toEqual([]); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("headless fallbacks count toward both tab limits and free capacity when closed", () => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + desktopAvailable = true; + const open = (agent: string) => + broker.invoke({ + scope: asSession(agent), + operation: "open", + input: { reuseExistingTab: false, show: false }, + }); + let first: PreviewAutomationStatus | undefined; + for (let agent = 0; agent < 4; agent++) { + for (let count = 0; count < 8; count++) { + const opened = yield* open(`agent-${agent}`); + first ??= opened; + } + expect(yield* open(`agent-${agent}`).pipe(Effect.flip)).toMatchObject({ + reason: "tabLimit", + }); + } + expect(yield* open("another-agent").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); + expect(contexts).toHaveLength(32); + expect( + (yield* manager.list({ threadId: scope.thread.threadId })).sessions.every( + (session) => session.serverPageMode === "headless", + ), + ).toBe(true); + desktopTabs.add(first!.tabId!); + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId: first!.tabId! }); + yield* broker.invoke({ + scope: asSession("agent-0"), + tabId: PreviewTabId.make(first!.tabId!), + operation: "close", + input: {}, + }); + expect((yield* open("another-agent")).available).toBe(true); + expect(desktopConnections).toEqual([]); + expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toHaveLength(32); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("a standalone server still opens a headless page when no desktop channel exists", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + expect(desktopAvailable).toBe(false); + expect(contexts).toHaveLength(1); + expect(desktopConnections).toEqual([]); + expect( + yield* broker.invoke({ scope, tabId, operation: "evaluate", input: { expression: "42" } }), + ).toBe("evaluated"); + }), + ).pipe(Effect.provide(layer)), +); + it.live("drives the desktop's own page for a tab the desktop renders", () => Effect.scoped( Effect.gen(function* () { @@ -1320,6 +1651,9 @@ it.live("drives the desktop's own page for a tab the desktop renders", () => desktopRendersNext = true; const opened = yield* manager.open({ threadId: scope.thread.threadId, runtime: "server" }); const viewer = yield* browser.attachViewer(viewerInput(opened.tabId, true)); + expect( + (yield* manager.list({ threadId: scope.thread.threadId })).sessions[0]?.serverPageMode, + ).toBe("native"); expect(desktopConnections.map((connection) => connection.endpoint)).toEqual([ `ws://desktop/${opened.tabId}`, ]); @@ -1569,22 +1903,22 @@ it.live("a desktop page that comes back reconnects without waiting for a viewer" ).pipe(Effect.provide(layer)), ); -it.live("an agent tab that went headless moves onto the desktop page that mounts late", () => +it.live("late desktop attachment keeps downloads and an active headless recording", () => Effect.scoped( Effect.gen(function* () { - const { browser, broker, tabId } = yield* ready; - const headless = contexts[0]!.page; - // The agent downloaded a file on the headless page; Chromium writes it, here the test does. + const { broker, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const page = contexts[0]!.page; const saved = Promise.withResolvers(); const written = Promise.withResolvers(); - headless.emit("download", { + page.emit("download", { failure: async () => null, saveAs: (path: string) => { saved.resolve(path); return written.promise; }, suggestedFilename: () => "report.csv", - url: () => "blob:http://localhost:5173/1", + url: () => "blob:http://localhost/1", }); const fs = yield* FileSystem.FileSystem; yield* fs.writeFileString(yield* Effect.promise(() => saved.promise), "a,b"); @@ -1596,32 +1930,74 @@ it.live("an agent tab that went headless moves onto the desktop page that mounts yield* Effect.yieldNow; before = yield* status(); } - // A remote viewer streams the headless page and has stopped reading. - const viewer = yield* browser.attachViewer(viewerInput(tabId, false)); - stallViewer(viewer, { _tag: "viewport", width: 1, height: 1 }); - while (desktopDetaches.listenerCount("attach") === 0) yield* Effect.yieldNow; - // The user opens the thread after the attach wait ran out, so the desktop mounts its page now. + yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); + const encoder = scratchPages[0]!; + const attached = Promise.withResolvers(); + const list = manager.list; + const spy = vi + .spyOn(manager, "list") + .mockImplementation((input) => + list(input).pipe(Effect.tap(() => Effect.sync(() => attached.resolve()))), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => spy.mockRestore())); desktopTabs.add(tabId); desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId }); - while (desktopConnections.length === 0) yield* Effect.yieldNow; - expect(headless.isClosed()).toBe(true); - // Even with a full queue, the viewer learns to reconnect to the new page. - expect(yield* Queue.take(viewer.output)).toEqual({ _tag: "reconnect" }); - yield* broker.invoke({ + // The stream processes this missing tab after the real attach, providing a completion barrier. + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId: "attach-barrier" }); + yield* Effect.promise(() => attached.promise); + expect(page.close).not.toHaveBeenCalled(); + expect(encoder.close).not.toHaveBeenCalled(); + expect(desktopConnections).toEqual([]); + expect((yield* status()).downloads).toEqual(before.downloads); + expect((yield* status()).serverPageMode).toBe("headless"); + encoder.evaluate + .mockResolvedValueOnce({ mimeType: "video/webm", bytes: 5, count: 1 }) + .mockResolvedValueOnce("dmlkZW8="); + const recording = yield* broker.invoke<{ path: string; sizeBytes: number }>({ scope, tabId, - operation: "navigate", - input: { url: "http://localhost:5173/next" }, + operation: "recordingStop", + input: {}, }); - // The agent now drives the page the user sees. - const desktopPage = desktopConnections[0]!.context.page; - expect(desktopPage.goto.mock.calls.map(([url]) => url)).toContain( - "http://localhost:5173/next", + expect(recording.sizeBytes).toBe(5); + expect(yield* fs.readFileString(recording.path)).toBe("video"); + expect(yield* fs.exists(before.downloads?.[0]?.path ?? "")).toBe(true); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("a capture timeout releases the tab so the next agent action works", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const cdp = contexts[0]!.sessions[0]!; + const started = Promise.withResolvers(); + const captured = Promise.withResolvers>(); + const send = cdp.send.getMockImplementation()!; + cdp.send.mockImplementation((method, input) => { + if (method !== "Page.captureScreenshot") return send(method, input); + started.resolve(); + return captured.promise; + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + captured.resolve({ data: "ZnJhbWU=" }); + vi.useRealTimers(); + }), ); - // The session stayed open, so the agent's download is still there. - const after = yield* status(); - expect(after.downloads).toEqual(before.downloads); - expect(yield* fs.exists(after.downloads?.[0]?.path ?? "")).toBe(true); + const snapshot = yield* broker + .invoke({ scope, tabId, operation: "snapshot", input: {} }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Effect.promise(() => started.promise); + yield* Effect.promise(() => vi.advanceTimersByTimeAsync(5_000)); + expect(yield* Fiber.join(snapshot)).toMatchObject({ + _tag: "PreviewAutomationExecutionError", + reason: expect.stringContaining("did not draw a frame"), + }); + expect( + yield* broker.invoke({ scope, tabId, operation: "evaluate", input: { expression: "42" } }), + ).toBe("evaluated"); }), ).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 388672d12d0a8..11e88dfc9beb7 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -683,16 +683,14 @@ const make = Effect.gen(function* () { /** Whether the preview session for a tab still exists. */ const sessionOpen = (tab: ServerTab) => !closedSessions.has(tabKey(tab.threadId, tab.tabId)); - /** `handOff` keeps the session for a new page, so viewers reconnect to it. */ - const dropTab = (tab: ServerTab, closeSession: boolean, handOff = false) => { + const dropTab = (tab: ServerTab, closeSession: boolean) => { const key = tabKey(tab.threadId, tab.tabId); if (tabs.get(key) !== tab) return; tabs.delete(key); tab.closing = true; clearAbortedNavigation(tab); // A desktop page outlives the connection unless its session closed with it. - const end = - (tab.desktop || handOff) && !closeSession && sessionOpen(tab) ? "reconnect" : "gone"; + const end = tab.desktop && !closeSession && sessionOpen(tab) ? "reconnect" : "gone"; for (const viewer of tab.viewers) viewer.push({ _tag: end }); void tab.control.close().catch(constVoid); // The desktop owns its page; letting go only ends this connection. @@ -700,9 +698,7 @@ const make = Effect.gen(function* () { else void tab.page.close().catch(constVoid); if (tab.isolatedContext) void tab.page.context().close().catch(constVoid); void tab.recording?.encoder.close().catch(constVoid); - // A handed-off tab keeps its downloads; the folder is keyed by tab, not page. - if (!handOff) - void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid); + void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid); reportLiveTabs(); if (closeSession) { runFork(manager.close({ threadId: tab.threadId, tabId: tab.tabId }).pipe(Effect.ignore)); @@ -710,11 +706,10 @@ const make = Effect.gen(function* () { }; /** - * With a desktop app attached, every tab of this server renders there, so a - * new tab waits for its `` instead of launching headless. + * Prefer the desktop's page, but keep a chosen headless page across reattachments. */ const desktopRenders = (snapshot: PreviewSessionSnapshot) => - desktopChannel.available + desktopChannel.available && snapshot.serverPageMode !== "headless" ? Effect.runPromise( desktopChannel.awaitAttached( { threadId: snapshot.threadId, tabId: snapshot.tabId }, @@ -746,13 +741,41 @@ const make = Effect.gen(function* () { } }; - const createTab = async (snapshot: PreviewSessionSnapshot): Promise => { - const adopted = adoptedPages.get(tabKey(snapshot.threadId, snapshot.tabId)); - adoptedPages.delete(tabKey(snapshot.threadId, snapshot.tabId)); - const desktop = - adopted === undefined && (await desktopRenders(snapshot)) - ? await connectDesktop(snapshot) - : null; + const currentSnapshot = async (threadId: string, tabId: string) => { + const { sessions } = await Effect.runPromise( + manager.list({ threadId: ThreadId.make(threadId) }), + ); + const snapshot = sessions.find( + (session) => session.tabId === tabId && session.runtime === "server", + ); + if (!snapshot) throw new ServerBrowserTabNotFoundError({ threadId, tabId }); + return snapshot; + }; + + const createTab = async (requestedSnapshot: PreviewSessionSnapshot): Promise => { + const key = tabKey(requestedSnapshot.threadId, requestedSnapshot.tabId); + const adopted = adoptedPages.get(key); + adoptedPages.delete(key); + // An older opened event may be processed after a failed headless launch or a close. + const fresh = await currentSnapshot(requestedSnapshot.threadId, requestedSnapshot.tabId).catch( + async (cause: unknown) => { + await adopted?.page.close().catch(constVoid); + throw cause; + }, + ); + let snapshot = fresh; + const native = adopted === undefined && (await desktopRenders(snapshot)); + if (adopted === undefined) { + // Publish before launching headless so every client releases the independent native guest. + snapshot = await Effect.runPromise( + manager.setServerPageMode({ + threadId: ThreadId.make(snapshot.threadId), + tabId: snapshot.tabId, + mode: native ? "native" : "headless", + }), + ); + } + const desktop = native ? await connectDesktop(snapshot) : null; // An agent tab without a profile (no client reported one) keeps throwaway storage. const isolatedContext = adopted === undefined && @@ -900,8 +923,8 @@ const make = Effect.gen(function* () { // Popups become tabs and keep `window.opener`, so sign-in popups can report back. page.on("popup", (popup) => void adoptPopup(tab, popup)); // Playwright cannot reload a crashed page, so it must leave the tab list. - const key = tabKey(tab.threadId, tab.tabId); - if (closedPendingTabs.delete(key)) { + const { sessions } = await Effect.runPromise(manager.list({ threadId: tab.threadId })); + if (closedPendingTabs.delete(key) || !sessions.some((session) => session.tabId === tab.tabId)) { await control.close().catch(constVoid); if (desktop) await desktop.close().catch(constVoid); else await page.close().catch(constVoid); @@ -1162,6 +1185,7 @@ const make = Effect.gen(function* () { threadId: opener.threadId, ...(/^https?:/i.test(url) ? { url } : {}), runtime: "server", + ...(!opener.desktop ? { serverPageMode: "headless" as const } : {}), ...(opener.profileId === undefined ? {} : { profileId: opener.profileId }), // Agent popups stay with the agent and only float when it asks, like its own opens. ...(opener.control.agentId === null @@ -1340,6 +1364,7 @@ const make = Effect.gen(function* () { url: url === "about:blank" ? null : url, title: null, loading: tab.loading, + serverPageMode: tab.desktop ? ("native" as const) : ("headless" as const), control: { owner: tabOwner(tab), ownedByCaller: tab.control.agentId === agentSessionId, @@ -1360,6 +1385,7 @@ const make = Effect.gen(function* () { owner: tabOwner(candidate), ownedByCaller: candidate.control.agentId === agentSessionId, visible: candidate.viewers.size > 0, + serverPageMode: candidate.desktop ? ("native" as const) : ("headless" as const), ...(candidate.profileId === undefined ? {} : { profileId: candidate.profileId }), })), ...profileStatus(), @@ -2417,28 +2443,16 @@ const make = Effect.gen(function* () { // The page came back, for example after its DevTools closed. Reconnect now // rather than on the next viewer or agent, or the tab's URL and title stop // reaching every client in the meantime. - // A tab that went headless before the desktop mounted it (its thread was in - // the background) moves onto the desktop's page. Otherwise the user and the - // agent see two different pages under one tab. yield* desktopChannel.attached.pipe( Stream.runForEach((key) => Effect.gen(function* () { - const pending = pendingTabs.get(tabKey(key.threadId, key.tabId)); - if (pending) yield* Effect.promise(() => pending.catch(constVoid)); - const existing = tabs.get(tabKey(key.threadId, key.tabId)); - if (existing?.desktop) return; - if (existing) dropTab(existing, false, true); + if (tabs.has(tabKey(key.threadId, key.tabId))) return; const { sessions } = yield* manager.list({ threadId: ThreadId.make(key.threadId) }); const snapshot = sessions.find( (session) => session.tabId === key.tabId && session.runtime === "server", ); - const tab = snapshot - ? yield* Effect.promise(() => ensureTab(snapshot).catch(() => null)) - : null; - if (!existing) return; - if (tab) tab.downloads.unshift(...existing.downloads); - else - void NodeFSP.rm(downloadDir(existing), { recursive: true, force: true }).catch(constVoid); + if (snapshot && snapshot.serverPageMode !== "headless") + yield* Effect.promise(() => ensureTab(snapshot).catch(constVoid)); }).pipe(Effect.ignore), ), Effect.forkScoped, diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 7ace8ecb79485..59da0d02e9ed0 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -152,7 +152,7 @@ const SNAPSHOT_SCRIPT = `(() => { const CAPTURE_TIMEOUT_MS = 5_000; const NO_FRAME_MESSAGE = - "The page did not draw a frame, so no screenshot was taken. In the desktop app, a tab only draws while the user has its thread open. Use preview_evaluate to read the page, or ask the user to open the thread."; + "The page did not draw a frame within 5 seconds, so no screenshot was taken. Use preview_evaluate to read the page, or open the tab in the Browser panel and retry."; /** Captures Chromium has not answered yet; a page only draws one at a time. */ const pendingCaptures = new WeakSet(); diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index 2e602201d6ec9..cc0a3258e7d88 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -15,7 +15,7 @@ import { previewEnvironment } from "~/state/preview"; import { useEnvironmentScope } from "~/state/session"; import { useAtomCommand } from "~/state/use-atom-command"; -import { holdFramesForTurn } from "./agentFrameRequests"; +import { holdFramesForCapture, releaseCaptureFrames } from "./agentFrameRequests"; import { readPreviewAnnotationTheme } from "./annotationTheme"; import { useBrowserDefaults } from "./browserDefaults"; import { useBrowserPointerStore } from "./browserPointerStore"; @@ -103,17 +103,23 @@ export function ElectronBrowserHost() { const openPreview = useAtomCommand(previewEnvironment.open, { reportFailure: true }); const sessionByRuntimeTabId = useRef(new Map()); useEffect(() => { - sessionByRuntimeTabId.current = new Map( - sessions.map((session) => [session.runtimeTabId, session]), - ); + const next = new Map(sessions.map((session) => [session.runtimeTabId, session])); + for (const tabId of sessionByRuntimeTabId.current.keys()) { + if (!next.has(tabId)) releaseCaptureFrames(tabId); + } + sessionByRuntimeTabId.current = next; }, [sessions]); useEffect(() => { const preview = window.desktopBridge?.preview; if (!preview) return; - return preview.onFrameRequest((tabId) => { - const source = sessionByRuntimeTabId.current.get(tabId); - if (source) holdFramesForTurn(tabId, source.threadRef); + const unsubscribe = preview.onFrameRequest((tabId) => { + const session = sessionByRuntimeTabId.current.get(tabId); + if (session) holdFramesForCapture(tabId, session.threadRef); }); + return () => { + unsubscribe(); + for (const tabId of sessionByRuntimeTabId.current.keys()) releaseCaptureFrames(tabId); + }; }, []); useEffect(() => { const preview = window.desktopBridge?.preview; diff --git a/apps/web/src/browser/HostedBrowserWebview.test.tsx b/apps/web/src/browser/HostedBrowserWebview.test.tsx index aa582251cb079..6ba86087e7539 100644 --- a/apps/web/src/browser/HostedBrowserWebview.test.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.test.tsx @@ -99,6 +99,51 @@ afterEach(async () => { }); describe("HostedBrowserWebview settings hydration", () => { + it("releases a removed native guest without registering it when tab creation finishes late", async () => { + mocks.getClientSettings.mockResolvedValue(DEFAULT_CLIENT_SETTINGS); + await act(async () => ensureClientSettingsHydrated()); + const tabCreation = deferred(); + mocks.createTab.mockReturnValueOnce(tabCreation.promise); + const threadRef = { + environmentId: EnvironmentId.make("headless-fallback"), + threadId: ThreadId.make("thread-fallback"), + }; + const runtimeTabId = "late-native-fallback"; + await act(() => { + renderer = create( + , + { + createNodeMock: (element) => + element.type === "webview" + ? Object.assign(new EventTarget(), { getWebContentsId: () => 41 }) + : { scrollLeft: 0, scrollTop: 0, scrollTo: () => undefined }, + }, + ); + }); + expect(mocks.createTab).toHaveBeenCalledOnce(); + expect(mocks.registerWebview).not.toHaveBeenCalled(); + vi.useFakeTimers(); + await act(() => renderer?.unmount()); + renderer = undefined; + await act(async () => { + tabCreation.resolve(); + await tabCreation.promise; + await vi.advanceTimersByTimeAsync(0); + }); + expect(mocks.registerWebview).not.toHaveBeenCalled(); + expect(mocks.closeTab).toHaveBeenCalledExactlyOnceWith(runtimeTabId); + }); + it("starts a retained background tab only after a settings read succeeds on retry", async () => { const firstRead = deferred(); const retryRead = deferred(); diff --git a/apps/web/src/browser/agentFrameRequests.test.ts b/apps/web/src/browser/agentFrameRequests.test.ts index e7f161fcd3217..b14f5b0fd329d 100644 --- a/apps/web/src/browser/agentFrameRequests.test.ts +++ b/apps/web/src/browser/agentFrameRequests.test.ts @@ -1,42 +1,81 @@ import { EnvironmentId, ThreadId } from "@t3tools/contracts"; import { Atom } from "effect/reactivity"; -import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; - import { appAtomRegistry } from "~/rpc/atomRegistry"; -import { holdFramesForTurn } from "./agentFrameRequests"; +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { holdFramesForCapture, releaseCaptureFrames } from "./agentFrameRequests"; import { useBrowserSurfaceStore } from "./browserSurfaceStore"; const shell = vi.hoisted(() => ({ atom: null as unknown })); vi.mock("~/state/threads", () => ({ environmentThreadShells: { threadShellAtom: () => shell.atom }, })); - type Shell = { readonly runtime: { readonly status: string } | null }; +const shellAtom = Atom.make(null).pipe(Atom.keepAlive); +shell.atom = shellAtom; const threadRef = { environmentId: EnvironmentId.make("environment-1"), threadId: ThreadId.make("thread-1"), }; -const shellAtom = Atom.make(null).pipe(Atom.keepAlive); -shell.atom = shellAtom; -const setStatus = (status: string) => appAtomRegistry.set(shellAtom, { runtime: { status } }); const drawing = (tabId: string) => useBrowserSurfaceStore.getState().activityByTabId[tabId] ?? 0; -describe("holdFramesForTurn", () => { - beforeEach(() => setStatus("idle")); +afterEach(() => { + releaseCaptureFrames("tab-1"); + vi.useRealTimers(); +}); + +describe("capture frame leases", () => { + it("keeps an active turn drawing until it ends, including after the capture limit", () => { + vi.useFakeTimers(); + appAtomRegistry.set(shellAtom, { runtime: { status: "running" } }); + holdFramesForCapture("tab-1", threadRef); + holdFramesForCapture("tab-1", threadRef); + vi.advanceTimersByTime(10_000); + expect(drawing("tab-1")).toBe(1); + appAtomRegistry.set(shellAtom, { runtime: { status: "idle" } }); + expect(drawing("tab-1")).toBe(0); + }); - it("keeps a hidden tab drawing until the thread's turn ends", () => { - setStatus("running"); - holdFramesForTurn("tab-1", threadRef); - // Later captures in the same turn reuse the lease. - holdFramesForTurn("tab-1", threadRef); + it("bounds the lease when the background thread shell is missing", () => { + vi.useFakeTimers(); + appAtomRegistry.set(shellAtom, null); + holdFramesForCapture("tab-1", threadRef); expect(drawing("tab-1")).toBe(1); - setStatus("idle"); + vi.advanceTimersByTime(5_000); expect(drawing("tab-1")).toBe(0); }); - it("does not draw for a capture outside a turn", () => { - holdFramesForTurn("tab-2", threadRef); - expect(drawing("tab-2")).toBe(0); + it("draws without thread state and stops after the capture limit", () => { + vi.useFakeTimers(); + holdFramesForCapture("tab-1"); + expect(drawing("tab-1")).toBe(1); + vi.advanceTimersByTime(5_000); + expect(drawing("tab-1")).toBe(0); + }); + + it("renews one lease for repeated captures", () => { + vi.useFakeTimers(); + holdFramesForCapture("tab-1"); + vi.advanceTimersByTime(4_000); + holdFramesForCapture("tab-1"); + expect(drawing("tab-1")).toBe(1); + vi.advanceTimersByTime(4_000); + expect(drawing("tab-1")).toBe(1); + vi.advanceTimersByTime(1_000); + expect(drawing("tab-1")).toBe(0); + }); + + it("releases a removed guest without releasing another activity lease", () => { + vi.useFakeTimers(); + const releaseRecording = useBrowserSurfaceStore.getState().acquireActivity("tab-1"); + holdFramesForCapture("tab-1"); + expect(drawing("tab-1")).toBe(2); + releaseCaptureFrames("tab-1"); + releaseCaptureFrames("tab-1"); + vi.advanceTimersByTime(5_000); + expect(drawing("tab-1")).toBe(1); + releaseRecording(); + expect(drawing("tab-1")).toBe(0); }); }); diff --git a/apps/web/src/browser/agentFrameRequests.ts b/apps/web/src/browser/agentFrameRequests.ts index 6fc060e6c5309..215e5984df225 100644 --- a/apps/web/src/browser/agentFrameRequests.ts +++ b/apps/web/src/browser/agentFrameRequests.ts @@ -6,25 +6,32 @@ import { environmentThreadShells } from "~/state/threads"; import { acquireBrowserSurfaceActivity } from "./browserSurfaceStore"; -const leases = new Map void>(); +// Matches the server's capture limit when the thread shell is not loaded. +const CAPTURE_FRAME_LEASE_MS = 5_000; +const leases = new Map void; unsubscribe: () => void }>(); -/** - * Keeps a hidden tab drawing after the server asked for one of its frames, until - * the thread's turn ends. A tab parked offscreen draws nothing, so an agent's - * screenshot would wait forever. Later requests in the same turn reuse the lease. - */ -export function holdFramesForTurn(tabId: string, threadRef: ScopedThreadRef): void { - if (leases.has(tabId)) return; - const shellAtom = environmentThreadShells.threadShellAtom(threadRef); - if (!threadRuntimeIsActive(appAtomRegistry.get(shellAtom)?.runtime)) return; - const releaseActivity = acquireBrowserSurfaceActivity(tabId); - const release = () => { - leases.delete(tabId); - unsubscribe(); - releaseActivity(); - }; - const unsubscribe = appAtomRegistry.subscribe(shellAtom, (shell) => { - if (!threadRuntimeIsActive(shell?.runtime)) release(); - }); - leases.set(tabId, release); +/** Keeps a hidden page drawing through an active turn, or for one bounded capture. */ +export function holdFramesForCapture(tabId: string, threadRef?: ScopedThreadRef): void { + const previous = leases.get(tabId); + previous?.unsubscribe(); + const release = previous?.release ?? acquireBrowserSurfaceActivity(tabId); + const shellAtom = threadRef ? environmentThreadShells.threadShellAtom(threadRef) : null; + if (shellAtom && threadRuntimeIsActive(appAtomRegistry.get(shellAtom)?.runtime)) { + const unsubscribe = appAtomRegistry.subscribe(shellAtom, (shell) => { + if (!threadRuntimeIsActive(shell?.runtime)) releaseCaptureFrames(tabId); + }); + leases.set(tabId, { release, unsubscribe }); + } else { + const timer = setTimeout(() => releaseCaptureFrames(tabId), CAPTURE_FRAME_LEASE_MS); + leases.set(tabId, { release, unsubscribe: () => clearTimeout(timer) }); + } +} + +/** Releases a frame lease when its native guest or host goes away. */ +export function releaseCaptureFrames(tabId: string): void { + const lease = leases.get(tabId); + if (!lease) return; + leases.delete(tabId); + lease.unsubscribe(); + lease.release(); } diff --git a/apps/web/src/browser/previewRuntime.test.ts b/apps/web/src/browser/previewRuntime.test.ts index 951b52e904687..9169e8216f3a5 100644 --- a/apps/web/src/browser/previewRuntime.test.ts +++ b/apps/web/src/browser/previewRuntime.test.ts @@ -7,7 +7,11 @@ const state = vi.hoisted(() => ({ serverBrowser: new Set(), })); -vi.mock("~/env", () => ({ isElectron: false })); +vi.mock("~/env", () => ({ + get isElectron() { + return state.desktop; + }, +})); vi.mock("~/previewStateStore", () => ({ isPreviewSupportedInRuntime: () => state.desktop })); vi.mock("~/rpc/atomRegistry", () => ({ appAtomRegistry: { get: () => state.primary } })); vi.mock("~/state/primaryEnvironment", () => ({ primaryEnvironmentIdAtom: {} })); @@ -16,7 +20,11 @@ vi.mock("~/state/entities", () => ({ useEnvironmentSupportsServerBrowser: () => false, })); -import { alternatePreviewRuntime, previewRuntimeFor } from "./previewRuntime"; +import { + alternatePreviewRuntime, + previewRuntimeFor, + rendersServerTabNatively, +} from "./previewRuntime"; const local = "local" as EnvironmentId; const remote = "remote" as EnvironmentId; @@ -58,3 +66,44 @@ describe("alternatePreviewRuntime", () => { expect(alternatePreviewRuntime(remote, null, true, { runtime: "server" })).toBeNull(); }); }); + +describe("rendersServerTabNatively", () => { + it("streams a local tab after headless fallback while retaining native and legacy tabs", () => { + state.desktop = true; + expect(rendersServerTabNatively(local, local, { runtime: "server" })).toBe(true); + expect( + rendersServerTabNatively(local, local, { + runtime: "server", + serverPageMode: "native", + }), + ).toBe(true); + expect( + rendersServerTabNatively(local, local, { + runtime: "server", + serverPageMode: "headless", + }), + ).toBe(false); + }); + + it("never hosts remote or non-server tabs as local server webviews", () => { + state.desktop = true; + expect( + rendersServerTabNatively(remote, local, { + runtime: "server", + serverPageMode: "native", + }), + ).toBe(false); + expect(rendersServerTabNatively(local, null, { runtime: "server" })).toBe(false); + expect(rendersServerTabNatively(local, local, { runtime: "desktop" })).toBe(false); + expect(rendersServerTabNatively(local, local, null)).toBe(false); + }); + + it("streams native server pages in a web client", () => { + expect( + rendersServerTabNatively(local, local, { + runtime: "server", + serverPageMode: "native", + }), + ).toBe(false); + }); +}); diff --git a/apps/web/src/browser/previewRuntime.ts b/apps/web/src/browser/previewRuntime.ts index 435eb07bbc3c5..1818e4720757f 100644 --- a/apps/web/src/browser/previewRuntime.ts +++ b/apps/web/src/browser/previewRuntime.ts @@ -56,16 +56,18 @@ export function usePreviewAvailable(environmentId: EnvironmentId | null): boolea /** * Whether this client draws a server tab with its own ``. The desktop * app renders tabs of the server it launched, which drives them over the - * desktop browser channel; every other client and environment streams them. + * desktop browser channel; headless tabs and every other client and environment + * stream them. */ export function rendersServerTabNatively( environmentId: EnvironmentId, primaryEnvironmentId: EnvironmentId | null, - snapshot: Pick | null | undefined, + snapshot: Pick | null | undefined, ): boolean { return ( isElectron && snapshot?.runtime === "server" && + snapshot.serverPageMode !== "headless" && primaryEnvironmentId !== null && environmentId === primaryEnvironmentId ); @@ -73,7 +75,7 @@ export function rendersServerTabNatively( export function useRendersServerTabNatively( environmentId: EnvironmentId, - snapshot: Pick | null | undefined, + snapshot: Pick | null | undefined, ): boolean { return rendersServerTabNatively(environmentId, useAtomValue(primaryEnvironmentIdAtom), snapshot); } diff --git a/apps/web/src/components/preview/PreviewView.test.tsx b/apps/web/src/components/preview/PreviewView.test.tsx index 09eb7ee8e1d50..7dbb3a8c3c18f 100644 --- a/apps/web/src/components/preview/PreviewView.test.tsx +++ b/apps/web/src/components/preview/PreviewView.test.tsx @@ -6,10 +6,19 @@ import { EnvironmentId, FILL_PREVIEW_VIEWPORT, ThreadId, + type PreviewSessionSnapshot, } from "@t3tools/contracts"; -import { act, createElement, Profiler } from "react"; +import { + act, + createElement, + Profiler, + useEffect, + useImperativeHandle, + type ComponentProps, +} from "react"; import { renderToStaticMarkup } from "react-dom/server"; import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import type { ServerBrowserSurface } from "~/browser/ServerBrowserSurface"; const mocks = vi.hoisted(() => ({ navigate: vi.fn(async (_tabId: string, _url: string): Promise => undefined), @@ -40,6 +49,62 @@ const mocks = vi.hoisted(() => ({ recordingTabIds: new Set(), recordingRuntimeTabId: null as string | null, recordVisitForThread: vi.fn(), + electron: false, + runtime: undefined as PreviewSessionSnapshot["runtime"], + serverPageMode: undefined as PreviewSessionSnapshot["serverPageMode"], + chromeLoading: false, + chromeInputDisabled: false, + nativeSurfaceMounted: vi.fn(), + nativeSurfaceUnmounted: vi.fn(), + streamMounted: vi.fn(), + streamUnmounted: vi.fn(), + streamNavigate: vi.fn(), +})); + +vi.mock("~/env", async (importOriginal) => ({ + ...(await importOriginal()), + get isElectron() { + return mocks.electron; + }, +})); +vi.mock("~/browser/previewRuntime", async (importOriginal) => { + const runtime = await importOriginal(); + return { + ...runtime, + useRendersServerTabNatively: ( + environmentId: EnvironmentId, + snapshot: Pick | null | undefined, + ) => + runtime.rendersServerTabNatively( + environmentId, + EnvironmentId.make("environment-1"), + snapshot, + ), + }; +}); +vi.mock("~/browser/ServerBrowserSurface", () => ({ + ServerBrowserSurface: (props: ComponentProps) => { + useImperativeHandle( + props.ref, + () => ({ + navigate: mocks.streamNavigate, + history: vi.fn(), + reload: vi.fn(), + viewport: vi.fn(), + canvas: () => null, + }), + [], + ); + useEffect(() => { + mocks.streamMounted(props.tabId); + props.onControl?.({ controller: "you", canOperate: true, generation: 1, dialog: null }); + props.onFirstFrame?.(); + return () => { + mocks.streamUnmounted(props.tabId); + }; + }, []); + return createElement("server-surface"); + }, })); const EMPTY_HISTORY: never[] = []; @@ -135,6 +200,8 @@ vi.mock("~/previewStateStore", () => ({ "tab-1": { threadId: "thread-1", tabId: "tab-1", + runtime: mocks.runtime, + serverPageMode: mocks.serverPageMode, navStatus: { _tag: "Success", url: "http://example.com/", @@ -232,6 +299,8 @@ vi.mock("./previewBridge", () => ({ vi.mock("./PreviewChromeRow", () => ({ PreviewChromeRow: (props: { onSubmit: (url: string) => void; + loading: boolean; + inputDisabled?: boolean; onPickElement?: () => void; onPictureInPicture?: () => void; pictureInPicture?: boolean; @@ -240,6 +309,8 @@ vi.mock("./PreviewChromeRow", () => ({ }; }) => { mocks.submittedUrl = props.onSubmit; + mocks.chromeLoading = props.loading; + mocks.chromeInputDisabled = props.inputDisabled ?? false; mocks.toggleAnnotation = props.onPickElement ?? null; mocks.togglePictureInPicture = props.onPictureInPicture ?? null; mocks.toggleNativePictureInPicture = @@ -266,7 +337,17 @@ vi.mock("./ZoomIndicator", () => ({ ZoomIndicator: () => null })); vi.mock("./AgentBrowserCursor", () => ({ AgentBrowserCursor: () => createElement("agent-cursor"), })); -vi.mock("~/browser/BrowserSurfaceSlot", () => ({ BrowserSurfaceSlot: () => null })); +vi.mock("~/browser/BrowserSurfaceSlot", () => ({ + BrowserSurfaceSlot: (props: { tabId: string }) => { + useEffect(() => { + mocks.nativeSurfaceMounted(props.tabId); + return () => { + mocks.nativeSurfaceUnmounted(props.tabId); + }; + }, [props.tabId]); + return null; + }, +})); vi.mock("./usePreviewSession", () => ({ usePreviewSession: vi.fn() })); import { PreviewView } from "./PreviewView"; @@ -372,6 +453,52 @@ describe("PreviewView navigation", () => { mocks.recordingTabIds = new Set(); mocks.recordingRuntimeTabId = null; mocks.recordVisitForThread.mockClear(); + mocks.electron = false; + mocks.runtime = undefined; + mocks.serverPageMode = undefined; + mocks.nativeSurfaceMounted.mockClear(); + mocks.nativeSurfaceUnmounted.mockClear(); + mocks.streamMounted.mockClear(); + mocks.streamUnmounted.mockClear(); + mocks.streamNavigate.mockClear(); + }); + + it("switches a native server tab to its stream and drops stale desktop state", async () => { + mocks.electron = true; + mocks.runtime = "server"; + mocks.serverPageMode = "native"; + mocks.loading = true; + const document = installTestDom(); + const { createRoot } = await import("react-dom/client"); + const container = document.createElement("div"); + const root = createRoot(container as unknown as Element); + const hasCursor = (node: TestNode): boolean => + node.nodeName === "AGENT-CURSOR" || node.childNodes.some(hasCursor); + const view = () => ; + try { + await act(() => root.render(view())); + expect(mocks.nativeSurfaceMounted).toHaveBeenCalledWith(TEST_RUNTIME_TAB_ID); + expect(mocks.streamMounted).not.toHaveBeenCalled(); + expect(mocks.chromeLoading).toBe(true); + expect(hasCursor(container)).toBe(true); + + mocks.serverPageMode = "headless"; + await act(() => root.render(view())); + expect(mocks.nativeSurfaceUnmounted).toHaveBeenCalledWith(TEST_RUNTIME_TAB_ID); + expect(mocks.streamMounted).toHaveBeenCalledTimes(1); + expect(mocks.streamMounted).toHaveBeenCalledWith("tab-1"); + expect(mocks.chromeLoading).toBe(false); + expect(mocks.chromeInputDisabled).toBe(false); + expect(mocks.toggleAnnotation).toBeNull(); + expect(hasCursor(container)).toBe(false); + + await act(async () => mocks.submittedUrl?.("localhost:3000/stream")); + expect(mocks.streamNavigate).toHaveBeenCalledWith("http://localhost:3000/stream"); + expect(mocks.navigate).not.toHaveBeenCalled(); + } finally { + await act(() => root.unmount()); + vi.unstubAllGlobals(); + } }); it("shows the cursor in a replacement browser while the old instance still records", async () => { diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index 161fc4c888c87..992391596f987 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -226,7 +226,8 @@ export function PreviewView({ const [serverControlledTabId, setServerControlledTabId] = useState(null); const serverInputDisabled = isServerTab && serverControlledTabId !== runtimeTabId; const [serverFrameTabId, setServerFrameTabId] = useState(null); - const desktopOverlay = tabId ? (previewState.desktopByTabId[tabId] ?? null) : null; + const desktopOverlay = + !isServerTab && tabId ? (previewState.desktopByTabId[tabId] ?? null) : null; const navStatus = snapshot?.navStatus ?? { _tag: "Idle" as const }; const url = navStatus._tag === "Idle" ? "" : navStatus.url; const loading = desktopOverlay?.loading ?? navStatus._tag === "Loading"; diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index b249c44afc108..d69f49e0a8fdd 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -125,12 +125,12 @@ function BrowserMiniPlayer({ threadRef, tabId, miniPlayer }: Props & { readonly const recording = recordingTabIds.has(runtimeTabId) || findActiveBrowserRecordingRuntimeTabId(threadRef, tabId) !== null; - const desktopOverlay = previewState.desktopByTabId[tabId] ?? null; const fittedSourceContent = useBrowserSurfaceStore( (state) => state.byTabId[runtimeTabId]?.fittedSourceContent ?? null, ); const nativeServerTab = useRendersServerTabNatively(threadRef.environmentId, snapshot); const serverTab = snapshot?.runtime === "server" && !nativeServerTab; + const desktopOverlay = serverTab ? null : (previewState.desktopByTabId[tabId] ?? null); const [streamViewport, setStreamViewport] = useState(null); const serverSurfaceRef = useRef(null); const serverPictureInPicture = @@ -140,7 +140,7 @@ function BrowserMiniPlayer({ threadRef, tabId, miniPlayer }: Props & { readonly ? streamViewport : resolvePreviewMiniPlayerSourceSize( snapshot?.viewport ?? FILL_PREVIEW_VIEWPORT, - fittedSourceContent, + serverTab ? null : fittedSourceContent, desktopOverlay?.zoomFactor ?? 1, ); diff --git a/apps/web/src/components/preview/previewFallback.test.tsx b/apps/web/src/components/preview/previewFallback.test.tsx new file mode 100644 index 0000000000000..177dc64975a94 --- /dev/null +++ b/apps/web/src/components/preview/previewFallback.test.tsx @@ -0,0 +1,227 @@ +import { scopedThreadKey } from "@t3tools/client-runtime/environment"; +import { EnvironmentId, ThreadId, type PreviewSessionSnapshot } from "@t3tools/contracts"; +import { + act, + createElement, + useEffect, + useImperativeHandle, + type ComponentProps, + type ReactNode, +} from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import type { HostedBrowserWebview } from "~/browser/HostedBrowserWebview"; +import type { ServerBrowserSurface } from "~/browser/ServerBrowserSurface"; + +const mocks = vi.hoisted(() => ({ + mode: "native" as "native" | "headless", + nativeMount: vi.fn(), + nativeUnmount: vi.fn(), + streamMount: vi.fn(), + streamUnmount: vi.fn(), + reportPreview: vi.fn(), + clearPreview: vi.fn(), + nativePopOut: vi.fn(), + nativeClosePopOut: vi.fn(), + serverPopOut: vi.fn(), +})); + +const threadRef = { + environmentId: EnvironmentId.make("local"), + threadId: ThreadId.make("thread-1"), +}; +function previewState() { + return { + serverEpoch: "server-1", + sessions: { + "tab-1": { + threadId: threadRef.threadId, + tabId: "tab-1", + runtime: "server", + serverPageMode: mocks.mode, + navStatus: { _tag: "Success", url: "https://example.com/", title: "Example" }, + viewport: { _tag: "freeform", width: 800, height: 600 }, + profileId: "default", + canGoBack: false, + canGoForward: false, + updatedAt: "2026-10-11T00:00:00.000Z", + } satisfies PreviewSessionSnapshot, + }, + desktopByTabId: { + "tab-1": { hasWebContents: true, zoomFactor: 2, pictureInPicture: true }, + }, + }; +} + +vi.mock("~/env", () => ({ isElectron: true })); +vi.mock("@effect/atom-react", () => ({ useAtomValue: () => threadRef.environmentId })); +vi.mock("~/state/primaryEnvironment", () => ({ primaryEnvironmentIdAtom: {} })); +vi.mock("~/state/entities", () => ({ + readEnvironmentSupportsServerBrowser: () => true, + useEnvironmentSupportsServerBrowser: () => true, +})); +vi.mock("~/previewStateStore", () => ({ + useThreadPreviewState: previewState, + useActivePreviewSessions: () => ({ [scopedThreadKey(threadRef)]: previewState() }), + isPreviewSupportedInRuntime: () => true, +})); +vi.mock("~/hooks/useSettings", () => ({ useClientSettingsHydrated: () => true })); +vi.mock("~/hooks/useTheme", () => ({ useTheme: () => ({ resolvedTheme: "light" }) })); +vi.mock("~/browser/browserDefaults", () => ({ + useBrowserDefaults: () => ({ profileId: "default" }), +})); +vi.mock("~/browser/browserPointerStore", () => ({ useBrowserPointerStore: {} })); +vi.mock("~/state/session", () => ({ useEnvironmentScope: () => true })); +vi.mock("~/state/preview", () => ({ previewEnvironment: { open: {} } })); +vi.mock("~/state/use-atom-command", () => ({ useAtomCommand: () => vi.fn() })); +vi.mock("~/browser/openFileInPreview", () => ({ openUrlInPreview: vi.fn() })); +vi.mock("~/browser/HostedBrowserWebview", () => ({ + HostedBrowserWebview: (props: ComponentProps) => { + useEffect(() => { + mocks.nativeMount(props.runtimeTabId); + return () => { + mocks.nativeUnmount(props.runtimeTabId); + }; + }, [props.runtimeTabId]); + return createElement("webview"); + }, +})); +vi.mock("~/browser/browserRecording", () => ({ + findActiveBrowserRecordingRuntimeTabId: () => null, + useActiveBrowserRecordingTabIds: () => new Set(), +})); +vi.mock("~/browser/browserSurfaceStore", () => ({ + useBrowserSurfaceStore: (select: (state: { byTabId: Record }) => unknown) => + select({ byTabId: {} }), +})); +vi.mock("~/browser/BrowserSurfaceSlot", () => ({ + BrowserSurfaceSlot: () => createElement("span", { "aria-label": "Native surface" }), +})); +vi.mock("~/browser/ServerBrowserSurface", () => ({ + ServerBrowserSurface: (props: ComponentProps) => { + useImperativeHandle( + props.ref, + () => ({ + navigate: vi.fn(), + history: vi.fn(), + reload: vi.fn(), + viewport: vi.fn(), + canvas: () => null, + }), + [], + ); + useEffect(() => { + mocks.streamMount(props.tabId); + return () => { + mocks.streamUnmount(props.tabId); + }; + }, [props.tabId]); + return createElement("stream-surface"); + }, +})); +vi.mock("~/browser/serverPictureInPicture", () => ({ + closeServerPictureInPicture: vi.fn(), + openServerPictureInPicture: mocks.serverPopOut, + serverPictureInPictureKey: (threadId: string, tabId: string) => `${threadId}:${tabId}`, + supportsServerPictureInPicture: () => true, + useServerPictureInPictureKey: () => null, +})); +vi.mock("./previewBridge", () => ({ + previewBridge: { pictureInPicture: { open: mocks.nativePopOut, close: mocks.nativeClosePopOut } }, +})); +vi.mock("../chat/ChatCanvasContext", () => ({ + useChatCanvas: () => ({ + previewKey: "browser:tab-1", + layout: { frame: { x: 0, y: 0, width: 240, height: 180 } }, + reportPreview: mocks.reportPreview, + clearPreview: mocks.clearPreview, + }), +})); +vi.mock("~/state/device", () => ({ + useDeviceState: () => ({ state: { devices: [], hosts: [] } }), +})); +vi.mock("../device/DeviceStreamView", () => ({ DeviceStreamView: () => null })); +vi.mock("~/components/ui/button", () => ({ + Button: (props: ComponentProps<"button">) => createElement("button", props), +})); +vi.mock("~/components/ui/tooltip", () => ({ + Tooltip: (props: { children: ReactNode }) => props.children, + TooltipTrigger: (props: { render: ReactNode }) => props.render, + TooltipPopup: () => null, +})); + +import { ElectronBrowserHost } from "~/browser/ElectronBrowserHost"; +import { ThreadPreviewMiniPlayer } from "./ThreadPreviewMiniPlayer"; +import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId"; + +let renderer: ReactTestRenderer | undefined; +beforeEach(() => { + vi.clearAllMocks(); + mocks.mode = "native"; + vi.stubGlobal("window", {}); + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); +}); +afterEach(async () => { + await act(() => renderer?.unmount()); + renderer = undefined; + vi.unstubAllGlobals(); +}); + +describe("headless fallback surfaces", () => { + it("unmounts the hosted native guest and never remounts it for a headless snapshot", async () => { + const runtimeTabId = previewRuntimeTabId(threadRef, "server-1", "tab-1"); + await act(() => { + renderer = create(); + }); + expect(mocks.nativeMount).toHaveBeenCalledExactlyOnceWith(runtimeTabId); + expect(renderer!.root.findAllByType("webview")).toHaveLength(1); + mocks.mode = "headless"; + await act(() => renderer!.update()); + expect(mocks.nativeUnmount).toHaveBeenCalledExactlyOnceWith(runtimeTabId); + expect(renderer!.root.findAllByType("webview")).toHaveLength(0); + await act(() => renderer!.update()); + expect(mocks.nativeMount).toHaveBeenCalledTimes(1); + }); + + it("moves a floating tab to its stream and uses server pop-out instead of stale native PiP", async () => { + const view = () => ( + + ); + await act(() => { + renderer = create(view()); + }); + expect(renderer!.root.findAllByProps({ "aria-label": "Native surface" })).toHaveLength(1); + const nativeCloseButton = renderer!.root.findByProps({ + "aria-label": "Close popped-out preview", + }); + await act(async () => nativeCloseButton.props.onClick()); + expect(mocks.nativeClosePopOut).toHaveBeenCalledOnce(); + + mocks.mode = "headless"; + await act(() => renderer!.update(view())); + expect(renderer!.root.findAllByProps({ "aria-label": "Native surface" })).toHaveLength(0); + expect(mocks.streamMount).toHaveBeenCalledExactlyOnceWith("tab-1"); + expect(mocks.reportPreview).toHaveBeenLastCalledWith( + expect.objectContaining({ source: { width: 800, height: 600 } }), + ); + const serverOpenButton = renderer!.root.findByProps({ + "aria-label": "Pop preview into separate window", + }); + await act(async () => serverOpenButton.props.onClick()); + expect(mocks.serverPopOut).toHaveBeenCalledExactlyOnceWith({ + ...threadRef, + tabId: "tab-1", + seed: null, + }); + expect(mocks.nativeClosePopOut).toHaveBeenCalledOnce(); + expect(mocks.nativePopOut).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/components/preview/usePreviewBridge.test.ts b/apps/web/src/components/preview/usePreviewBridge.test.ts index 14acf0d69609e..31fe7863c5bca 100644 --- a/apps/web/src/components/preview/usePreviewBridge.test.ts +++ b/apps/web/src/components/preview/usePreviewBridge.test.ts @@ -1,7 +1,69 @@ -import type { DesktopPreviewTabState } from "@t3tools/contracts"; -import { describe, expect, it } from "vite-plus/test"; +import { EnvironmentId, ThreadId, type DesktopPreviewTabState } from "@t3tools/contracts"; +import * as Option from "effect/Option"; +import { act, createElement } from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -import { projectDesktopState } from "./usePreviewBridge"; +const mocks = vi.hoisted(() => ({ + mode: undefined as "native" | "headless" | undefined, + listener: null as ((tabId: string, state: DesktopPreviewTabState) => void) | null, + applyDesktopState: vi.fn(), + unsubscribe: vi.fn(), + reportStatus: vi.fn(), + clearPointer: vi.fn(), +})); + +vi.mock("~/previewStateStore", () => ({ + applyPreviewDesktopState: mocks.applyDesktopState, + readThreadPreviewState: () => ({ sessions: { "tab-1": { serverPageMode: mocks.mode } } }), +})); +vi.mock("~/browserFaviconStore", () => ({ + flushPendingFaviconsForThread: vi.fn(), + recordFaviconForThread: vi.fn(), + useFaviconProjectRefForThread: () => null, +})); +vi.mock("~/browser/browserPointerStore", () => ({ + useBrowserPointerStore: (select: (state: { clear: typeof mocks.clearPointer }) => unknown) => + select({ clear: mocks.clearPointer }), +})); +vi.mock("~/state/preview", () => ({ previewEnvironment: { reportStatus: {} } })); +vi.mock("~/state/session", () => ({ usePreparedConnection: () => Option.none() })); +vi.mock("~/state/use-atom-command", () => ({ useAtomCommand: () => mocks.reportStatus })); +vi.mock("./previewBridge", () => ({ + previewBridge: { + onStateChange: (listener: typeof mocks.listener) => { + mocks.listener = listener; + return mocks.unsubscribe; + }, + }, +})); + +import { projectDesktopState, usePreviewBridge } from "./usePreviewBridge"; + +const threadRef = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), +}; +let renderer: ReactTestRenderer | undefined; + +function BridgeHarness() { + usePreviewBridge({ threadRef, tabId: "tab-1", runtimeTabId: "native-tab-1", serverDriven: true }); + return null; +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.mode = undefined; + mocks.listener = null; + vi.stubGlobal("window", {}); + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); +}); + +afterEach(async () => { + await act(() => renderer?.unmount()); + renderer = undefined; + vi.unstubAllGlobals(); +}); const favicon = { dataUrl: "data:image/png;base64,AAAA", @@ -48,3 +110,50 @@ describe("projectDesktopState", () => { expect(projectDesktopState(state({ kind: "Idle" })).favicon).toBeNull(); }); }); + +describe("native bridge lifetime", () => { + it("drops late native state when the server switches this tab to headless", async () => { + mocks.mode = "native"; + await act(() => { + renderer = create(createElement(BridgeHarness)); + }); + mocks.listener?.( + "native-tab-1", + state({ kind: "Loading", url: "https://example.com/", title: "" }), + ); + expect(mocks.applyDesktopState).toHaveBeenLastCalledWith( + threadRef, + "tab-1", + expect.objectContaining({ loading: true }), + ); + expect(mocks.reportStatus).not.toHaveBeenCalled(); + + mocks.mode = "headless"; + mocks.listener?.( + "native-tab-1", + state({ kind: "Success", url: "https://other.example/", title: "Late" }), + ); + expect(mocks.applyDesktopState).toHaveBeenLastCalledWith(threadRef, "tab-1", null); + expect(mocks.reportStatus).not.toHaveBeenCalled(); + + await act(() => renderer?.unmount()); + renderer = undefined; + expect(mocks.unsubscribe).toHaveBeenCalledTimes(1); + expect(mocks.applyDesktopState).toHaveBeenLastCalledWith(threadRef, "tab-1", null); + }); + + it("clears the native overlay on headless unmount even without another native event", async () => { + await act(() => { + renderer = create(createElement(BridgeHarness)); + }); + mocks.listener?.( + "native-tab-1", + state({ kind: "Loading", url: "https://example.com/", title: "" }), + ); + mocks.mode = "headless"; + await act(() => renderer?.unmount()); + renderer = undefined; + expect(mocks.unsubscribe).toHaveBeenCalledTimes(1); + expect(mocks.applyDesktopState).toHaveBeenLastCalledWith(threadRef, "tab-1", null); + }); +}); diff --git a/apps/web/src/components/preview/usePreviewBridge.ts b/apps/web/src/components/preview/usePreviewBridge.ts index ee737892a58d0..83772b35b3b97 100644 --- a/apps/web/src/components/preview/usePreviewBridge.ts +++ b/apps/web/src/components/preview/usePreviewBridge.ts @@ -16,7 +16,11 @@ import { useFaviconProjectRefForThread, } from "~/browserFaviconStore"; import { useBrowserPointerStore } from "~/browser/browserPointerStore"; -import { applyPreviewDesktopState, type DesktopPreviewOverlay } from "~/previewStateStore"; +import { + applyPreviewDesktopState, + readThreadPreviewState, + type DesktopPreviewOverlay, +} from "~/previewStateStore"; import { previewEnvironment } from "~/state/preview"; import { usePreparedConnection } from "~/state/session"; import { useAtomCommand } from "~/state/use-atom-command"; @@ -66,6 +70,11 @@ export function usePreviewBridge(input: { const handleStateChange = useEffectEvent( (changedTabId: string, state: DesktopPreviewTabState): void => { if (changedTabId !== runtimeTabId) return; + // A late native guest must not override the headless page's stream state. + if (readThreadPreviewState(stableThreadRef).sessions[tabId]?.serverPageMode === "headless") { + applyPreviewDesktopState(stableThreadRef, tabId, null); + return; + } if (shouldClearBrowserPointer(lastDesktopNavStatus.current, state.navStatus)) { clearBrowserPointer(runtimeTabId); } @@ -95,7 +104,13 @@ export function usePreviewBridge(input: { lastReportedUrl.current = null; lastReportedKind.current = null; lastDesktopNavStatus.current = null; - return bridge.onStateChange(handleStateChange); + const unsubscribe = bridge.onStateChange(handleStateChange); + return () => { + unsubscribe(); + if (readThreadPreviewState(stableThreadRef).sessions[tabId]?.serverPageMode === "headless") { + applyPreviewDesktopState(stableThreadRef, tabId, null); + } + }; }, [bridge, runtimeTabId, stableThreadRef, tabId]); useEffect(() => { if (!projectRef) return; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 7b86c3131365c..1a8645061af1e 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1352,7 +1352,7 @@ export interface DesktopPreviewBridge { }; onStateChange: (listener: (tabId: string, state: DesktopPreviewTabState) => void) => () => void; onPointerEvent: (listener: (event: DesktopPreviewPointerEvent) => void) => () => void; - /** The server waits for a frame of this hidden tab, so it should draw until the turn ends. */ + /** The server waits for a frame of this hidden tab; keep it drawing for the capture limit. */ onFrameRequest: (listener: (tabId: string) => void) => () => void; onOpenLink: (listener: (event: DesktopPreviewOpenLinkEvent) => void) => () => void; } diff --git a/packages/contracts/src/preview.test.ts b/packages/contracts/src/preview.test.ts index 24f429745ef87..707ef3d5f777a 100644 --- a/packages/contracts/src/preview.test.ts +++ b/packages/contracts/src/preview.test.ts @@ -80,21 +80,45 @@ describe("PreviewNavStatus", () => { }); describe("PreviewSessionSnapshot", () => { - it("round-trips a Success snapshot", () => { + const legacySnapshot = { + threadId: "thread-1", + tabId: "preview-thread-1", + navStatus: { + _tag: "Success" as const, + url: "http://localhost:5173/", + title: "Vite App", + }, + canGoBack: false, + canGoForward: false, + updatedAt: "2026-01-01T00:00:00.000Z", + }; + const jsonCodec = Schema.toCodecJson(PreviewSessionSnapshot); + const encodeJson = Schema.encodeSync(jsonCodec); + const decodeJson = Schema.decodeUnknownSync(jsonCodec); + + it("round-trips a legacy snapshot without choosing a page backing", () => { + const snapshot = decodeJson(encodeJson(decodeSnapshot(legacySnapshot))); + expect(snapshot).toEqual(legacySnapshot); + expect(snapshot).not.toHaveProperty("serverPageMode"); + }); + + it.each(["native", "headless"] as const)("preserves %s server page backing over JSON", (mode) => { const snapshot = decodeSnapshot({ - threadId: "thread-1", - tabId: "preview-thread-1", - navStatus: { - _tag: "Success", - url: "http://localhost:5173/", - title: "Vite App", - }, - canGoBack: false, - canGoForward: false, - updatedAt: "2026-01-01T00:00:00.000Z", + ...legacySnapshot, + runtime: "server", + serverPageMode: mode, + }); + expect(decodeJson(encodeJson(snapshot))).toEqual({ + ...legacySnapshot, + runtime: "server", + serverPageMode: mode, }); - expect(snapshot.tabId).toBe("preview-thread-1"); - expect(snapshot.navStatus._tag).toBe("Success"); + }); + + it("rejects a runtime value used as a page backing", () => { + expect(() => + decodeSnapshot({ ...legacySnapshot, runtime: "server", serverPageMode: "server" }), + ).toThrow(); }); }); @@ -244,6 +268,31 @@ describe("PreviewEvent", () => { expect(event.type).toBe("opened"); }); + it("preserves the headless fallback in a snapshot update sent to clients", () => { + const event = decodePreviewEvent({ + type: "navigated", + threadId: "t", + tabId: "preview-t", + createdAt: "2026-01-01T00:00:00.000Z", + serverEpoch: "server-a", + revision: 2, + snapshot: { + threadId: "t", + tabId: "preview-t", + runtime: "server", + serverPageMode: "headless", + navStatus: { _tag: "Loading", url: "https://example.com/", title: "" }, + canGoBack: false, + canGoForward: false, + updatedAt: "2026-01-01T00:00:00.000Z", + }, + }); + const jsonCodec = Schema.toCodecJson(PreviewEvent); + const received = Schema.decodeUnknownSync(jsonCodec)(Schema.encodeSync(jsonCodec)(event)); + expect(received).toEqual(event); + expect(received.type === "navigated" && received.snapshot.serverPageMode).toBe("headless"); + }); + it("decodes failed with code/description", () => { const event = decodePreviewEvent({ type: "failed", diff --git a/packages/contracts/src/preview.ts b/packages/contracts/src/preview.ts index 802bbd4765b7c..2d61108b0fb9a 100644 --- a/packages/contracts/src/preview.ts +++ b/packages/contracts/src/preview.ts @@ -172,10 +172,10 @@ export const PreviewNavStatus = Schema.Union([ export type PreviewNavStatus = typeof PreviewNavStatus.Type; /** - * Where a tab's page runs. `desktop` is an Electron owned by one - * desktop client; `server` is headless Chromium owned by the environment - * server, viewed by any client through `/api/preview-stream` and driven by - * agents with no client attached. Absent means `desktop`. + * Who manages a tab. `desktop` is an Electron owned by one desktop + * client; `server` is owned by the environment server and driven by agents. + * A server tab can use its desktop host's native page or headless Chromium; + * other clients view either through `/api/preview-stream`. Absent means `desktop`. */ export const PreviewRuntime = Schema.Literals(["desktop", "server"]); export type PreviewRuntime = typeof PreviewRuntime.Type; @@ -213,6 +213,8 @@ export const PreviewSessionSnapshot = Schema.Struct({ */ profileId: Schema.optional(BrowserProfileId), runtime: Schema.optional(PreviewRuntime), + /** Server-selected page backing. Absent preserves clients' existing host-based rendering choice. */ + serverPageMode: Schema.optional(Schema.Literals(["native", "headless"])), /** Authenticated provider session owning an isolated server tab. */ automationOwner: Schema.optional(Schema.String), /** An agent opened this tab and asked to show it, so viewers float it. */ diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 504d28148b296..410e5e8014379 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -92,6 +92,8 @@ export const PreviewAutomationStatus = Schema.Struct({ url: Schema.NullOr(Schema.String), title: Schema.NullOr(Schema.String), loading: Schema.Boolean, + /** Actual server page backing. Older hosts may omit it. */ + serverPageMode: Schema.optional(Schema.Literals(["native", "headless"])), control: Schema.optional( Schema.Struct({ owner: PreviewAutomationTabOwner, @@ -143,6 +145,7 @@ export const PreviewAutomationStatus = Schema.Struct({ ownedByCaller: Schema.optional(Schema.Boolean), visible: Schema.optional(Schema.Boolean), profileId: Schema.optional(BrowserProfileId), + serverPageMode: Schema.optional(Schema.Literals(["native", "headless"])), }), ), ),