diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index a6b617385b578..a8f07bac29b1e 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -111,6 +111,7 @@ export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save"; export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame"; export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change"; export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event"; +export const PREVIEW_FRAME_REQUEST_CHANNEL = "desktop:preview-frame-request"; export const PREVIEW_OPEN_LINK_CHANNEL = "desktop:preview-open-link"; export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 94c56464f7c47..980cffdc855c3 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -54,6 +54,9 @@ export const installPreviewEventForwarding = Effect.fn( yield* manager.subscribePointerEvents((event) => electronWindow.sendAll(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, event), ); + yield* manager.subscribeFrameRequests((tabId) => + electronWindow.sendAll(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, tabId), + ); yield* manager.subscribeOpenLinks((event) => electronWindow.sendAll(IpcChannels.PREVIEW_OPEN_LINK_CHANNEL, event), ); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index efd49ffa38648..90136c5703a12 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -429,6 +429,14 @@ contextBridge.exposeInMainWorld("desktopBridge", { return () => ipcRenderer.removeListener(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, wrappedListener); }, + onFrameRequest: (listener) => { + const wrappedListener = (_event: Electron.IpcRendererEvent, tabId: unknown) => { + if (typeof tabId === "string") listener(tabId); + }; + ipcRenderer.on(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener); + return () => + ipcRenderer.removeListener(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener); + }, onOpenLink: (listener) => { const wrappedListener = (_event: Electron.IpcRendererEvent, linkEvent: unknown) => { if (typeof linkEvent !== "object" || linkEvent === null) return; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 81ea040701f0e..c1d3a6510e30f 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -148,4 +148,25 @@ describe("DesktopBrowserHost", () => { expect(host.humanStartedDownload(debuggee.tab.webContents)).toBe(false); }), ); + + it.effect("asks the tab to draw when the server waits for a capture", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + host.attach(key, makeDebuggee().tab); + const requests = yield* host.frameRequests.pipe( + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Effect.yieldNow; + const command = (id: number, method: string) => + host.handleCommandLine( + encodeJson({ type: "cdp", ...key, message: encodeJson({ id, method, params: {} }) }), + ); + // Reading the page needs no frame; a capture waits for the next one. + yield* command(1, "DOM.getDocument"); + yield* command(2, "Page.captureScreenshot"); + expect(yield* Fiber.join(requests)).toEqual([key]); + }).pipe(Effect.scoped), + ); }); diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index cfb1ff753c470..1d8cfb902879b 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -31,6 +31,8 @@ const lineEncoder = new TextEncoder(); /** CDP commands through which an agent clicks, types, or navigates the page. */ const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; +/** A capture waits for the page's next frame, which a hidden tab only draws on request. */ +const CAPTURE_COMMAND = /"method":"Page\.captureScreenshot"/; /** How long after an agent's input a download still counts as the agent's. */ const AGENT_DOWNLOAD_WINDOW_MS = 5_000; @@ -93,6 +95,8 @@ export class DesktopBrowserHost extends Context.Service< readonly x: number; readonly y: number; }>; + /** Server tabs whose page the server is waiting to capture, so they must draw. */ + readonly frameRequests: Stream.Stream; } >()("@t3tools/desktop/preview/DesktopBrowserHost") {} @@ -104,6 +108,7 @@ export const make = Effect.gen(function* () { readonly x: number; readonly y: number; }>(16); + const frameRequests = yield* PubSub.sliding(16); const runFork = Effect.runForkWith(yield* Effect.context()); const tabs = new Map(); const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event)); @@ -204,6 +209,8 @@ export const make = Effect.gen(function* () { return; } if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now(); + if (CAPTURE_COMMAND.test(command.value.message)) + runFork(PubSub.publish(frameRequests, tab.key)); relayFor(tab).receive(command.value.message); }); @@ -226,6 +233,7 @@ export const make = Effect.gen(function* () { return DesktopBrowserHost.of({ pointers: Stream.fromPubSub(pointers), + frameRequests: Stream.fromPubSub(frameRequests), // Subscribes before announcing, so no attach falls between the two. events: Stream.unwrap( Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index aa36055452eb8..ddf1c437238e0 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -415,6 +415,8 @@ type RecordingInputListener = (event: DesktopPreviewRecordingInputEvent) => Effe type PointerEventListener = (event: DesktopPreviewPointerEvent) => Effect.Effect; +type FrameRequestListener = (tabId: string) => Effect.Effect; + type OpenLinkListener = (event: DesktopPreviewOpenLinkEvent) => Effect.Effect; /** @@ -562,6 +564,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const attachedRef = yield* Ref.make>(new Map()); const listenersRef = yield* Ref.make>(new Set()); const pointerEventListenersRef = yield* Ref.make>(new Set()); + const frameRequestListenersRef = yield* Ref.make>(new Set()); const openLinkListenersRef = yield* Ref.make>(new Set()); const recordingInputListenersRef = yield* Ref.make>( new Set(), @@ -838,6 +841,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function | "recording-frame" | "recording-input" | "pointer-event" + | "frame-request" | "open-link", tabId: string, delivery: () => Effect.Effect, @@ -2411,7 +2415,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return; } if (afterAttach.serverTab) { - yield* listenForAgentPointers; + yield* listenForAgentSignals; browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger }); } if (afterAttach.colorScheme !== "system") { @@ -3343,17 +3347,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { discard: true }, ); }); - // Listening starts with the first tab of the desktop's own server. - let pointersStarted = false; - const listenForAgentPointers = Effect.suspend(() => { - if (pointersStarted) return Effect.void; - pointersStarted = true; - return browserHost.pointers.pipe( - Stream.runForEach(emitAgentPointer), - Effect.forkIn(parentScope), - Effect.asVoid, + /** Tells the renderer that the server waits for a frame of this tab, so it must draw. */ + const emitFrameRequest = Effect.fn("PreviewManager.emitFrameRequest")(function* (key: { + readonly threadId: string; + readonly tabId: string; + }) { + const tab = [...(yield* SynchronizedRef.get(tabsRef)).values()].find( + (candidate) => + candidate.serverTab?.threadId === key.threadId && candidate.serverTab.tabId === key.tabId, + ); + if (!tab) return; + const listeners = yield* Ref.get(frameRequestListenersRef); + yield* Effect.forEach( + listeners, + (listener) => deliverEvent("frame-request", tab.tabId, () => listener(tab.tabId)), + { discard: true }, ); }); + // Listening starts with the first tab of the desktop's own server. + let agentSignalsStarted = false; + const listenForAgentSignals = Effect.suspend(() => { + if (agentSignalsStarted) return Effect.void; + agentSignalsStarted = true; + return Effect.all([ + browserHost.pointers.pipe(Stream.runForEach(emitAgentPointer), Effect.forkIn(parentScope)), + browserHost.frameRequests.pipe( + Stream.runForEach(emitFrameRequest), + Effect.forkIn(parentScope), + ), + ]).pipe(Effect.asVoid); + }); const emitOpenLink = Effect.fn("PreviewManager.emitOpenLink")(function* ( event: DesktopPreviewOpenLinkEvent, @@ -3386,6 +3409,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function [ Ref.set(listenersRef, new Set()), Ref.set(pointerEventListenersRef, new Set()), + Ref.set(frameRequestListenersRef, new Set()), Ref.set(openLinkListenersRef, new Set()), Ref.set(recordingFrameListenersRef, new Set()), Ref.set(recordingInputListenersRef, new Set()), @@ -3432,6 +3456,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function stopRecording, subscribePointerEvents: (listener: PointerEventListener) => subscribe(pointerEventListenersRef, listener), + subscribeFrameRequests: (listener: FrameRequestListener) => + subscribe(frameRequestListenersRef, listener), subscribeOpenLinks: (listener: OpenLinkListener) => subscribe(openLinkListenersRef, listener), subscribeRecordingInputs: (listener: RecordingInputListener) => subscribe(recordingInputListenersRef, listener), @@ -3674,6 +3700,9 @@ export class PreviewManager extends Context.Service< readonly subscribePointerEvents: ( listener: PointerEventListener, ) => Effect.Effect; + readonly subscribeFrameRequests: ( + listener: FrameRequestListener, + ) => Effect.Effect; readonly subscribeOpenLinks: ( listener: OpenLinkListener, ) => Effect.Effect; @@ -3819,6 +3848,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { saveRecording: operations.saveRecording, subscribeStateChanges: operations.subscribeStateChanges, subscribePointerEvents: operations.subscribePointerEvents, + subscribeFrameRequests: operations.subscribeFrameRequests, subscribeOpenLinks: operations.subscribeOpenLinks, subscribeRecordingFrames: operations.subscribeRecordingFrames, subscribeRecordingInputs: operations.subscribeRecordingInputs, diff --git a/apps/server/src/preview/Manager.test.ts b/apps/server/src/preview/Manager.test.ts index 48657d85d9023..96587a128e566 100644 --- a/apps/server/src/preview/Manager.test.ts +++ b/apps/server/src/preview/Manager.test.ts @@ -100,6 +100,89 @@ it.layer(layer)("PreviewManager", (it) => { }), ); + it.effect("publishes the server's page mode once and preserves a headless choice", () => + Effect.gen(function* () { + const threadId = freshThreadId(); + const manager = yield* PreviewManager.PreviewManager; + const collector = yield* collectEvents; + const opened = yield* manager.open({ + threadId, + runtime: "server", + url: "http://localhost:5173/", + profileId: "work", + automationOwner: "agent", + }); + const native = yield* manager.setServerPageMode({ + threadId, + tabId: opened.tabId, + mode: "native", + }); + const headless = yield* manager.setServerPageMode({ + threadId, + tabId: opened.tabId, + mode: "headless", + }); + expect(native.serverPageMode).toBe("native"); + expect(headless).toMatchObject({ + ...opened, + serverPageMode: "headless", + updatedAt: expect.any(String), + }); + const before = yield* manager.list({ threadId }); + for (const mode of ["headless", "native"] as const) { + expect(yield* manager.setServerPageMode({ threadId, tabId: opened.tabId, mode })).toEqual( + headless, + ); + } + const after = yield* manager.list({ threadId }); + expect(after).toEqual(before); + const events = yield* collector.drain; + expect(events.map((event) => event.type)).toEqual(["opened", "resized", "resized"]); + expect(events.at(-1)).toMatchObject({ snapshot: headless, revision: after.revision }); + yield* manager.reportStatus({ + threadId, + tabId: opened.tabId, + serverControlled: true, + navStatus: { _tag: "Success", url: "http://localhost:5173/next", title: "Next" }, + canGoBack: true, + canGoForward: false, + }); + yield* manager.resize({ threadId, tabId: opened.tabId, viewport: { _tag: "fill" } }); + expect((yield* manager.list({ threadId })).sessions[0]?.serverPageMode).toBe("headless"); + }), + ); + + it.effect( + "allows only server sessions to select a page mode, and never recreates a closed tab", + () => + Effect.gen(function* () { + const threadId = freshThreadId(); + const manager = yield* PreviewManager.PreviewManager; + const desktop = yield* manager.open({ threadId, serverPageMode: "headless" }); + expect(desktop).not.toHaveProperty("serverPageMode"); + expect( + (yield* manager + .setServerPageMode({ threadId, tabId: desktop.tabId, mode: "headless" }) + .pipe(Effect.flip))._tag, + ).toBe("PreviewControlRequiredError"); + const server = yield* manager.open({ + threadId, + runtime: "server", + serverPageMode: "headless", + }); + expect(server.serverPageMode).toBe("headless"); + yield* manager.close({ threadId, tabId: server.tabId }); + expect( + (yield* manager + .setServerPageMode({ threadId, tabId: server.tabId, mode: "headless" }) + .pipe(Effect.flip))._tag, + ).toBe("PreviewSessionLookupError"); + expect( + (yield* manager.list({ threadId })).sessions.map((session) => session.tabId), + ).toEqual([desktop.tabId]); + }), + ); + it.effect("lets any client size, theme, and zoom a server tab", () => Effect.gen(function* () { const threadId = freshThreadId(); diff --git a/apps/server/src/preview/Manager.ts b/apps/server/src/preview/Manager.ts index 9420e07a5a3a7..a3d79f5502020 100644 --- a/apps/server/src/preview/Manager.ts +++ b/apps/server/src/preview/Manager.ts @@ -48,6 +48,8 @@ export class PreviewManager extends Context.Service< readonly open: ( input: PreviewOpenInput & { readonly automationOwner?: string; + /** Only the server browser selects how its page is presented. */ + readonly serverPageMode?: PreviewSessionSnapshot["serverPageMode"]; /** Runs before the `opened` event publishes, so subscribers find state keyed by the tab. */ readonly beforePublish?: (snapshot: PreviewSessionSnapshot) => void; }, @@ -58,6 +60,13 @@ export class PreviewManager extends Context.Service< readonly reportStatus: ( input: PreviewReportStatusInput & { readonly serverControlled?: boolean }, ) => Effect.Effect; + /** Publishes the backing page selected by the server; headless cannot be promoted later. */ + readonly setServerPageMode: ( + input: PreviewCloseInput & { + readonly tabId: string; + readonly mode: "native" | "headless"; + }, + ) => Effect.Effect; readonly requestReveal: ( input: PreviewCloseInput & { readonly tabId: string; readonly force: boolean }, ) => Effect.Effect; @@ -221,6 +230,9 @@ export const make = Effect.gen(function* PreviewManagerMake() { viewport: input.viewport ?? FILL_PREVIEW_VIEWPORT, ...(input.profileId === undefined ? {} : { profileId: input.profileId }), ...(runtime === undefined ? {} : { runtime }), + ...(runtime === "server" && input.serverPageMode !== undefined + ? { serverPageMode: input.serverPageMode } + : {}), ...(runtime === "server" && input.automationOwner !== undefined ? { automationOwner: input.automationOwner } : {}), @@ -337,6 +349,40 @@ export const make = Effect.gen(function* PreviewManagerMake() { ); }); + const setServerPageMode: PreviewManager["Service"]["setServerPageMode"] = Effect.fn( + "PreviewManager.setServerPageMode", + )(function* (input) { + return yield* mutateExistingSession( + input.threadId, + input.tabId, + Effect.fn("PreviewManager.setSessionPageMode")(function* (session) { + if (session.snapshot.runtime !== "server") + return yield* new PreviewControlRequiredError({ tabId: input.tabId }); + if ( + session.snapshot.serverPageMode === input.mode || + session.snapshot.serverPageMode === "headless" + ) + return { next: session, emit: null, result: session.snapshot }; + const snapshot: PreviewSessionSnapshot = { + ...session.snapshot, + serverPageMode: input.mode, + updatedAt: yield* currentIsoTimestamp, + }; + return { + next: { ...session, snapshot }, + emit: { + type: "resized", + threadId: session.threadId, + tabId: session.tabId, + createdAt: snapshot.updatedAt, + snapshot, + }, + result: snapshot, + }; + }), + ); + }); + const resize: PreviewManager["Service"]["resize"] = Effect.fn("PreviewManager.resize")( function* (input) { return yield* mutateExistingSession( @@ -456,7 +502,8 @@ export const make = Effect.gen(function* PreviewManagerMake() { const list: PreviewManager["Service"]["list"] = Effect.fn("PreviewManager.list")( function* (input) { - return yield* SynchronizedRef.get(stateRef).pipe( + // Events publish inside a mutation; their consumers must wait for its state commit. + return yield* SynchronizedRef.modify(stateRef, (state) => [state, state] as const).pipe( Effect.map((state): PreviewListResult => ({ sessions: sessionsForThread(state, input.threadId) .map((s) => s.snapshot) @@ -504,6 +551,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { requestReveal, navigate, reportStatus, + setServerPageMode, resize, adjust, refresh, diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 385223a42d397..b92f35887bd98 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -40,6 +40,7 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ } async contextFor(profileId: string, isolationKey?: string) { contextRequests.push({ profileId, isolated: isolationKey !== undefined }); + contextRequested?.resolve(); if (contextFailure) throw contextFailure; await contextGate?.promise; const context = makeContext(this.onClose); @@ -47,7 +48,9 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ return context as unknown as BrowserContext; } async scratchPage() { - return makeContext().page as unknown as Page; + const page = makeContext().page; + scratchPages.push(page); + return page as unknown as Page; } async connectDesktopPage(endpoint: string) { const context = makeContext(); @@ -106,7 +109,7 @@ function makeContext(onClose?: (context: BrowserContext) => void) { reload: vi.fn(async () => {}), emulateMedia: vi.fn(async () => {}), waitForLoadState: vi.fn(async () => {}), - evaluate: vi.fn(async () => ({ + evaluate: vi.fn(async (): Promise => ({ url, title: "test page", loading: false, @@ -150,14 +153,21 @@ function makeContext(onClose?: (context: BrowserContext) => void) { } const contexts: ReturnType[] = []; +const scratchPages: ReturnType["page"][] = []; /** The profile and isolation each headless tab asked its context for. */ const contextRequests: Array<{ profileId: string; isolated: boolean }> = []; let contextGate: PromiseWithResolvers | null = null; +let contextRequested: PromiseWithResolvers | null = null; type ClipboardBinding = (source: { page: unknown }, text: unknown) => void; let clipboardBinding: ClipboardBinding | null = null; let contextFailure: Error | null = null; /** Server tabs the fake desktop renders, and the endpoints the server connected to. */ let desktopRendersNext = false; +let desktopAvailable = false; +let desktopAttachGate: PromiseWithResolvers | null = null; +let desktopAttachStarted: PromiseWithResolvers | null = null; +let desktopDetachConsumed: PromiseWithResolvers | null = null; +const desktopAttachRequests: string[] = []; /** Pages the fake desktop takes back or returns; the channel's streams emit them. */ const desktopDetaches = new NodeEvents.EventEmitter(); const desktopTabs = new Set(); @@ -169,7 +179,10 @@ const desktopRenders = (tabId: string) => { return desktopTabs.has(tabId); }; const releasedDesktopTabs: Array = []; -const desktopConnections: Array<{ endpoint: string; context: ReturnType }> = []; +const desktopConnections: Array<{ + endpoint: string; + context: ReturnType; +}> = []; const testThread = { threadId: ThreadId.make("browser-test-thread"), providerSessionId: "agent-a", @@ -200,9 +213,16 @@ const dependencies = Layer.mergeAll( installed: Effect.die("mock Chromium does not need an executable"), }), Layer.succeed(DesktopChannel.DesktopBrowserChannel, { - // Only tabs a test marks render on the desktop; the rest stay headless. - available: true, - awaitAttached: (key) => Effect.sync(() => desktopRenders(key.tabId)), + // Standalone tests have no desktop; native tests explicitly enable it. + get available() { + return desktopAvailable || desktopRendersNext || desktopTabs.size > 0; + }, + awaitAttached: (key) => + Effect.promise(async () => { + desktopAttachRequests.push(key.tabId); + desktopAttachStarted?.resolve(); + return desktopAttachGate ? desktopAttachGate.promise : desktopRenders(key.tabId); + }), detached: Stream.callback<{ threadId: string; tabId: string }>((queue) => Effect.acquireRelease( Effect.sync(() => { @@ -213,7 +233,7 @@ const dependencies = Layer.mergeAll( }), (onDetach) => Effect.sync(() => desktopDetaches.off("detach", onDetach)), ), - ), + ).pipe(Stream.tap(() => Effect.sync(() => desktopDetachConsumed?.resolve()))), attached: Stream.callback<{ threadId: string; tabId: string }>((queue) => Effect.acquireRelease( Effect.sync(() => { @@ -269,11 +289,18 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({ beforeEach(() => { contexts.length = 0; + scratchPages.length = 0; contextRequests.length = 0; contextGate = null; + contextRequested = null; contextFailure = null; desktopTabs.clear(); desktopRendersNext = false; + desktopAvailable = false; + desktopAttachGate = null; + desktopAttachStarted = null; + desktopDetachConsumed = null; + desktopAttachRequests.length = 0; releasedDesktopTabs.length = 0; desktopConnections.length = 0; }); @@ -960,12 +987,14 @@ it.live("a popup from a person's click is shown to that person, with its opener const browser = yield* ServerBrowser.ServerBrowser; const manager = yield* Manager.PreviewManager; yield* Effect.yieldNow; + desktopAvailable = true; const opened = yield* manager.open({ threadId: scope.thread.threadId, runtime: "server" }); const watcher = yield* browser.attachViewer(viewerInput(opened.tabId, false)); const viewer = yield* browser.attachViewer(viewerInput(opened.tabId, true)); yield* viewer.input({ type: "takeControl" }); const opener = contexts[0]!.page; const popup = makeContext(); + const events = yield* manager.subscribeEvents; opener.emit("popup", popup.page); let shown = yield* Queue.take(viewer.output); while (shown._tag !== "popup") shown = yield* Queue.take(viewer.output); @@ -973,6 +1002,16 @@ it.live("a popup from a person's click is shown to that person, with its opener expect(sessions.map((session) => session.tabId)).toEqual( expect.arrayContaining([opened.tabId, shown.tabId]), ); + expect(sessions.find((session) => session.tabId === shown.tabId)?.serverPageMode).toBe( + "headless", + ); + expect( + (yield* PubSub.takeUpTo(events, 10)).find((event) => event.type === "opened"), + ).toMatchObject({ snapshot: { tabId: shown.tabId, serverPageMode: "headless" } }); + yield* browser.attachViewer(viewerInput(shown.tabId, false)); + expect(desktopAttachRequests).toEqual([opened.tabId]); + expect(desktopConnections).toEqual([]); + expect(contexts).toHaveLength(1); expect((yield* Queue.clear(watcher.output)).some((item) => item._tag === "popup")).toBe( false, ); @@ -981,6 +1020,48 @@ it.live("a popup from a person's click is shown to that person, with its opener ).pipe(Effect.provide(layer)), ); +it.live("a popup closed before adoption does not leave its page behind", () => + Effect.scoped( + Effect.gen(function* () { + const { tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const events = yield* manager.subscribeEvents; + const reading = Promise.withResolvers(); + const read = Promise.withResolvers(); + const closed = Promise.withResolvers(); + const list = manager.list; + const spy = vi.spyOn(manager, "list").mockImplementationOnce((input) => + Effect.promise(async () => { + reading.resolve(); + await read.promise; + }).pipe(Effect.andThen(list(input))), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + read.resolve(); + spy.mockRestore(); + }), + ); + const popup = makeContext(); + popup.page.once("close", () => closed.resolve()); + contexts[0]!.page.emit("popup", popup.page); + yield* Effect.promise(() => reading.promise); + const opened = yield* PubSub.take(events); + expect(opened.type).toBe("opened"); + yield* manager.close({ threadId: scope.thread.threadId, tabId: opened.tabId }); + read.resolve(); + yield* Effect.promise(() => closed.promise); + expect(popup.page.isClosed()).toBe(true); + expect(contexts[0]!.page.isClosed()).toBe(false); + expect( + (yield* manager.list({ threadId: scope.thread.threadId })).sessions.map( + (session) => session.tabId, + ), + ).toEqual([tabId]); + }), + ).pipe(Effect.provide(layer)), +); + it.live("closing a tab while a viewer is still opening it does not leave its page behind", () => Effect.scoped( Effect.gen(function* () { @@ -1310,6 +1391,256 @@ it.live("viewers see the agent's pointer move to its target and click there", () ).pipe(Effect.provide(layer)), ); +it.live( + "publishes a headless fallback before launch and keeps the same page after late native events", + () => + Effect.scoped( + Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + const events = yield* manager.subscribeEvents; + yield* Effect.yieldNow; + desktopAvailable = true; + const launching = Promise.withResolvers(); + const launch = Promise.withResolvers(); + contextRequested = launching; + contextGate = launch; + yield* Effect.addFinalizer(() => Effect.sync(() => launch.resolve())); + const opening = yield* broker + .invoke({ + scope, + operation: "open", + input: { url: "http://localhost:5173/fallback", reuseExistingTab: false, show: false }, + }) + .pipe(Effect.forkScoped); + yield* Effect.promise(() => launching.promise); + const listed = yield* manager.list({ threadId: scope.thread.threadId }); + const snapshot = listed.sessions[0]!; + expect(snapshot.serverPageMode).toBe("headless"); + expect(contexts).toEqual([]); + expect( + (yield* PubSub.takeUpTo(events, 10)).map((event) => ({ + type: event.type, + mode: "snapshot" in event ? event.snapshot.serverPageMode : undefined, + })), + ).toEqual([ + { type: "opened", mode: undefined }, + { type: "resized", mode: "headless" }, + ]); + // A native guest arriving while headless is still launching must not become another page. + const queried = Promise.withResolvers(); + const list = manager.list; + const spy = vi + .spyOn(manager, "list") + .mockImplementation((input) => + list(input).pipe(Effect.tap(() => Effect.sync(() => queried.resolve()))), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => spy.mockRestore())); + desktopTabs.add(snapshot.tabId); + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId: snapshot.tabId }); + yield* Effect.promise(() => queried.promise); + launch.resolve(); + const opened = yield* Fiber.join(opening); + expect(opened).toMatchObject({ + available: true, + tabId: snapshot.tabId, + url: "http://localhost:5173/fallback", + }); + expect(contexts).toHaveLength(1); + expect(contexts[0]!.page.goto).toHaveBeenCalledOnce(); + expect(desktopConnections).toEqual([]); + const viewer = yield* browser.attachViewer(viewerInput(snapshot.tabId, false)); + expect((yield* Queue.clear(viewer.output)).some((item) => item._tag === "frame")).toBe( + true, + ); + const consumed = Promise.withResolvers(); + desktopDetachConsumed = consumed; + desktopTabs.delete(snapshot.tabId); + desktopDetaches.emit("detach", { threadId: scope.thread.threadId, tabId: snapshot.tabId }); + yield* Effect.promise(() => consumed.promise); + expect( + yield* broker.invoke({ + scope, + tabId: PreviewTabId.make(snapshot.tabId), + operation: "evaluate", + input: { expression: "42" }, + }), + ).toBe("evaluated"); + expect(contexts[0]!.page.close).not.toHaveBeenCalled(); + expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toHaveLength(1); + }), + ).pipe(Effect.provide(layer)), +); + +it.live( + "a chosen headless page stays headless when its failed launch is retried after native attachment", + () => + Effect.scoped( + Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + desktopAvailable = true; + contextFailure = new Error("launch failed"); + yield* broker + .invoke({ + scope, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }) + .pipe(Effect.flip); + const snapshot = (yield* manager.list({ threadId: scope.thread.threadId })).sessions[0]!; + expect(snapshot.serverPageMode).toBe("headless"); + contextFailure = null; + desktopTabs.add(snapshot.tabId); + const viewer = yield* browser.attachViewer(viewerInput(snapshot.tabId, false)); + expect((yield* Queue.clear(viewer.output)).some((item) => item._tag === "frame")).toBe( + true, + ); + expect(desktopConnections).toEqual([]); + expect(contexts).toHaveLength(1); + expect(desktopAttachRequests).toEqual([snapshot.tabId]); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("closing a tab during its native wait prevents a headless fallback", () => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + desktopAvailable = true; + const waiting = Promise.withResolvers(); + const attached = Promise.withResolvers(); + desktopAttachStarted = waiting; + desktopAttachGate = attached; + yield* Effect.addFinalizer(() => Effect.sync(() => attached.resolve(false))); + const opening = yield* broker + .invoke({ + scope, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Effect.promise(() => waiting.promise); + const snapshot = (yield* manager.list({ threadId: scope.thread.threadId })).sessions[0]!; + yield* manager.close({ threadId: scope.thread.threadId, tabId: snapshot.tabId }); + attached.resolve(false); + yield* Fiber.join(opening); + expect(contextRequests).toEqual([]); + expect(contexts).toEqual([]); + expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toEqual([]); + }), + ).pipe(Effect.provide(layer)), +); + +it.live( + "closing a tab while headless launches discards its page and prevents late native resurrection", + () => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + desktopAvailable = true; + const launching = Promise.withResolvers(); + const launch = Promise.withResolvers(); + contextRequested = launching; + contextGate = launch; + yield* Effect.addFinalizer(() => Effect.sync(() => launch.resolve())); + const opening = yield* broker + .invoke({ + scope, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Effect.promise(() => launching.promise); + const snapshot = (yield* manager.list({ threadId: scope.thread.threadId })).sessions[0]!; + yield* manager.close({ threadId: scope.thread.threadId, tabId: snapshot.tabId }); + launch.resolve(); + yield* Fiber.join(opening); + expect(contexts[0]!.page.isClosed()).toBe(true); + expect(contexts[0]!.close).toHaveBeenCalledOnce(); + expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toEqual([]); + const queried = Promise.withResolvers(); + const list = manager.list; + const spy = vi + .spyOn(manager, "list") + .mockImplementation((input) => + list(input).pipe(Effect.tap(() => Effect.sync(() => queried.resolve()))), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => spy.mockRestore())); + desktopTabs.add(snapshot.tabId); + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId: snapshot.tabId }); + yield* Effect.promise(() => queried.promise); + expect(desktopConnections).toEqual([]); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("headless fallbacks count toward both tab limits and free capacity when closed", () => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* Broker.PreviewAutomationBroker; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + desktopAvailable = true; + const open = (agent: string) => + broker.invoke({ + scope: asSession(agent), + operation: "open", + input: { reuseExistingTab: false, show: false }, + }); + let first: PreviewAutomationStatus | undefined; + for (let agent = 0; agent < 4; agent++) { + for (let count = 0; count < 8; count++) { + const opened = yield* open(`agent-${agent}`); + first ??= opened; + } + expect(yield* open(`agent-${agent}`).pipe(Effect.flip)).toMatchObject({ + reason: "tabLimit", + }); + } + expect(yield* open("another-agent").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); + expect(contexts).toHaveLength(32); + expect( + (yield* manager.list({ threadId: scope.thread.threadId })).sessions.every( + (session) => session.serverPageMode === "headless", + ), + ).toBe(true); + desktopTabs.add(first!.tabId!); + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId: first!.tabId! }); + yield* broker.invoke({ + scope: asSession("agent-0"), + tabId: PreviewTabId.make(first!.tabId!), + operation: "close", + input: {}, + }); + expect((yield* open("another-agent")).available).toBe(true); + expect(desktopConnections).toEqual([]); + expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toHaveLength(32); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("a standalone server still opens a headless page when no desktop channel exists", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + expect(desktopAvailable).toBe(false); + expect(contexts).toHaveLength(1); + expect(desktopConnections).toEqual([]); + expect( + yield* broker.invoke({ scope, tabId, operation: "evaluate", input: { expression: "42" } }), + ).toBe("evaluated"); + }), + ).pipe(Effect.provide(layer)), +); + it.live("drives the desktop's own page for a tab the desktop renders", () => Effect.scoped( Effect.gen(function* () { @@ -1320,6 +1651,9 @@ it.live("drives the desktop's own page for a tab the desktop renders", () => desktopRendersNext = true; const opened = yield* manager.open({ threadId: scope.thread.threadId, runtime: "server" }); const viewer = yield* browser.attachViewer(viewerInput(opened.tabId, true)); + expect( + (yield* manager.list({ threadId: scope.thread.threadId })).sessions[0]?.serverPageMode, + ).toBe("native"); expect(desktopConnections.map((connection) => connection.endpoint)).toEqual([ `ws://desktop/${opened.tabId}`, ]); @@ -1568,3 +1902,102 @@ it.live("a desktop page that comes back reconnects without waiting for a viewer" }), ).pipe(Effect.provide(layer)), ); + +it.live("late desktop attachment keeps downloads and an active headless recording", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const page = contexts[0]!.page; + const saved = Promise.withResolvers(); + const written = Promise.withResolvers(); + page.emit("download", { + failure: async () => null, + saveAs: (path: string) => { + saved.resolve(path); + return written.promise; + }, + suggestedFilename: () => "report.csv", + url: () => "blob:http://localhost/1", + }); + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString(yield* Effect.promise(() => saved.promise), "a,b"); + written.resolve(); + const status = () => + broker.invoke({ scope, tabId, operation: "status", input: {} }); + let before = yield* status(); + while (!before.downloads?.length) { + yield* Effect.yieldNow; + before = yield* status(); + } + yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); + const encoder = scratchPages[0]!; + const attached = Promise.withResolvers(); + const list = manager.list; + const spy = vi + .spyOn(manager, "list") + .mockImplementation((input) => + list(input).pipe(Effect.tap(() => Effect.sync(() => attached.resolve()))), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => spy.mockRestore())); + desktopTabs.add(tabId); + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId }); + // The stream processes this missing tab after the real attach, providing a completion barrier. + desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId: "attach-barrier" }); + yield* Effect.promise(() => attached.promise); + expect(page.close).not.toHaveBeenCalled(); + expect(encoder.close).not.toHaveBeenCalled(); + expect(desktopConnections).toEqual([]); + expect((yield* status()).downloads).toEqual(before.downloads); + expect((yield* status()).serverPageMode).toBe("headless"); + encoder.evaluate + .mockResolvedValueOnce({ mimeType: "video/webm", bytes: 5, count: 1 }) + .mockResolvedValueOnce("dmlkZW8="); + const recording = yield* broker.invoke<{ path: string; sizeBytes: number }>({ + scope, + tabId, + operation: "recordingStop", + input: {}, + }); + expect(recording.sizeBytes).toBe(5); + expect(yield* fs.readFileString(recording.path)).toBe("video"); + expect(yield* fs.exists(before.downloads?.[0]?.path ?? "")).toBe(true); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("a capture timeout releases the tab so the next agent action works", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const cdp = contexts[0]!.sessions[0]!; + const started = Promise.withResolvers(); + const captured = Promise.withResolvers>(); + const send = cdp.send.getMockImplementation()!; + cdp.send.mockImplementation((method, input) => { + if (method !== "Page.captureScreenshot") return send(method, input); + started.resolve(); + return captured.promise; + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + captured.resolve({ data: "ZnJhbWU=" }); + vi.useRealTimers(); + }), + ); + const snapshot = yield* broker + .invoke({ scope, tabId, operation: "snapshot", input: {} }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Effect.promise(() => started.promise); + yield* Effect.promise(() => vi.advanceTimersByTimeAsync(5_000)); + expect(yield* Fiber.join(snapshot)).toMatchObject({ + _tag: "PreviewAutomationExecutionError", + reason: expect.stringContaining("did not draw a frame"), + }); + expect( + yield* broker.invoke({ scope, tabId, operation: "evaluate", input: { expression: "42" } }), + ).toBe("evaluated"); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 939d112c49081..11e88dfc9beb7 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -706,11 +706,10 @@ const make = Effect.gen(function* () { }; /** - * With a desktop app attached, every tab of this server renders there, so a - * new tab waits for its `` instead of launching headless. + * Prefer the desktop's page, but keep a chosen headless page across reattachments. */ const desktopRenders = (snapshot: PreviewSessionSnapshot) => - desktopChannel.available + desktopChannel.available && snapshot.serverPageMode !== "headless" ? Effect.runPromise( desktopChannel.awaitAttached( { threadId: snapshot.threadId, tabId: snapshot.tabId }, @@ -742,13 +741,41 @@ const make = Effect.gen(function* () { } }; - const createTab = async (snapshot: PreviewSessionSnapshot): Promise => { - const adopted = adoptedPages.get(tabKey(snapshot.threadId, snapshot.tabId)); - adoptedPages.delete(tabKey(snapshot.threadId, snapshot.tabId)); - const desktop = - adopted === undefined && (await desktopRenders(snapshot)) - ? await connectDesktop(snapshot) - : null; + const currentSnapshot = async (threadId: string, tabId: string) => { + const { sessions } = await Effect.runPromise( + manager.list({ threadId: ThreadId.make(threadId) }), + ); + const snapshot = sessions.find( + (session) => session.tabId === tabId && session.runtime === "server", + ); + if (!snapshot) throw new ServerBrowserTabNotFoundError({ threadId, tabId }); + return snapshot; + }; + + const createTab = async (requestedSnapshot: PreviewSessionSnapshot): Promise => { + const key = tabKey(requestedSnapshot.threadId, requestedSnapshot.tabId); + const adopted = adoptedPages.get(key); + adoptedPages.delete(key); + // An older opened event may be processed after a failed headless launch or a close. + const fresh = await currentSnapshot(requestedSnapshot.threadId, requestedSnapshot.tabId).catch( + async (cause: unknown) => { + await adopted?.page.close().catch(constVoid); + throw cause; + }, + ); + let snapshot = fresh; + const native = adopted === undefined && (await desktopRenders(snapshot)); + if (adopted === undefined) { + // Publish before launching headless so every client releases the independent native guest. + snapshot = await Effect.runPromise( + manager.setServerPageMode({ + threadId: ThreadId.make(snapshot.threadId), + tabId: snapshot.tabId, + mode: native ? "native" : "headless", + }), + ); + } + const desktop = native ? await connectDesktop(snapshot) : null; // An agent tab without a profile (no client reported one) keeps throwaway storage. const isolatedContext = adopted === undefined && @@ -896,8 +923,8 @@ const make = Effect.gen(function* () { // Popups become tabs and keep `window.opener`, so sign-in popups can report back. page.on("popup", (popup) => void adoptPopup(tab, popup)); // Playwright cannot reload a crashed page, so it must leave the tab list. - const key = tabKey(tab.threadId, tab.tabId); - if (closedPendingTabs.delete(key)) { + const { sessions } = await Effect.runPromise(manager.list({ threadId: tab.threadId })); + if (closedPendingTabs.delete(key) || !sessions.some((session) => session.tabId === tab.tabId)) { await control.close().catch(constVoid); if (desktop) await desktop.close().catch(constVoid); else await page.close().catch(constVoid); @@ -1158,6 +1185,7 @@ const make = Effect.gen(function* () { threadId: opener.threadId, ...(/^https?:/i.test(url) ? { url } : {}), runtime: "server", + ...(!opener.desktop ? { serverPageMode: "headless" as const } : {}), ...(opener.profileId === undefined ? {} : { profileId: opener.profileId }), // Agent popups stay with the agent and only float when it asks, like its own opens. ...(opener.control.agentId === null @@ -1336,6 +1364,7 @@ const make = Effect.gen(function* () { url: url === "about:blank" ? null : url, title: null, loading: tab.loading, + serverPageMode: tab.desktop ? ("native" as const) : ("headless" as const), control: { owner: tabOwner(tab), ownedByCaller: tab.control.agentId === agentSessionId, @@ -1356,6 +1385,7 @@ const make = Effect.gen(function* () { owner: tabOwner(candidate), ownedByCaller: candidate.control.agentId === agentSessionId, visible: candidate.viewers.size > 0, + serverPageMode: candidate.desktop ? ("native" as const) : ("headless" as const), ...(candidate.profileId === undefined ? {} : { profileId: candidate.profileId }), })), ...profileStatus(), @@ -2233,7 +2263,12 @@ const make = Effect.gen(function* () { if (next._tag === "frame") runFork(next.ack); // State a stalled viewer cannot miss replaces its backlog. It runs // synchronously so an older replacement can never land after a newer one. - else if (next._tag === "gone" || next._tag === "control" || next._tag === "fileChooser") { + else if ( + next._tag === "gone" || + next._tag === "reconnect" || + next._tag === "control" || + next._tag === "fileChooser" + ) { const dropped = Effect.runSyncExit(Queue.clear(output)); if (dropped._tag === "Failure") return; Queue.offerUnsafe(output, next); @@ -2416,7 +2451,8 @@ const make = Effect.gen(function* () { const snapshot = sessions.find( (session) => session.tabId === key.tabId && session.runtime === "server", ); - if (snapshot) yield* Effect.promise(() => ensureTab(snapshot).catch(constVoid)); + if (snapshot && snapshot.serverPageMode !== "headless") + yield* Effect.promise(() => ensureTab(snapshot).catch(constVoid)); }).pipe(Effect.ignore), ), Effect.forkScoped, diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 9a4775bb47173..e397a4a40556d 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -5,7 +5,16 @@ import { type CDPSession, type Page, } from "playwright-core"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vite-plus/test"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vite-plus/test"; import { presentAsChrome } from "./ServerBrowserContexts.ts"; import * as ServerBrowserPage from "./ServerBrowserPage.ts"; @@ -57,6 +66,39 @@ describe("server browser element refs", () => { }; const repeatedRows = `
    ${Array.from({ length: 5 }, (_, i) => `
  • row ${i + 1}
    • `).join("")}
`; + it("gives up on a screenshot when the page draws no frame", async () => { + // A desktop tab whose thread is not open never answers a capture. + let captures = 0; + const stuck = { + send: (method: string, params?: object) => { + if (method !== "Page.captureScreenshot") + return cdp.send(method as "Page.getLayoutMetrics", params); + captures += 1; + return new Promise(() => {}); + }, + } as unknown as CDPSession; + const snapshotOf = () => + ServerBrowserPage.snapshot({ + page, + cdp: stuck, + renderScale: 1, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const rejected = expect(snapshotOf()).rejects.toThrow("did not draw a frame"); + await vi.advanceTimersByTimeAsync(5_000); + await rejected; + // Chromium still holds the first capture, so a retry fails without starting another. + await expect(snapshotOf()).rejects.toThrow("did not draw a frame"); + expect(captures).toBe(1); + } finally { + vi.useRealTimers(); + } + }); + it("clicks the fifth repeated delete control without touching row one", async () => { await page.setContent(repeatedRows); const result = await takeSnapshot(); diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 3df4901f469c6..59da0d02e9ed0 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -150,8 +150,13 @@ const SNAPSHOT_SCRIPT = `(() => { }; })()`; -// Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets. -export const captureViewport = async ( +const CAPTURE_TIMEOUT_MS = 5_000; +const NO_FRAME_MESSAGE = + "The page did not draw a frame within 5 seconds, so no screenshot was taken. Use preview_evaluate to read the page, or open the tab in the Browser panel and retry."; +/** Captures Chromium has not answered yet; a page only draws one at a time. */ +const pendingCaptures = new WeakSet(); + +const captureFrame = async ( page: Page, cdp: CDPSession, options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, @@ -175,6 +180,31 @@ export const captureViewport = async ( return data; }; +/** + * Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets. + * A capture waits for the page's next frame. A desktop tab whose thread is not + * open draws none, so the capture gives up after a limit instead of holding + * the tab. A new capture is refused until Chromium answers the old one, which + * still owns the page's capture viewport. + */ +export const captureViewport = ( + page: Page, + cdp: CDPSession, + options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number }, +): Promise => { + const noFrame = () => + new ServerBrowserOperationError("PreviewAutomationExecutionError", NO_FRAME_MESSAGE); + if (pendingCaptures.has(page)) return Promise.reject(noFrame()); + pendingCaptures.add(page); + const capture = captureFrame(page, cdp, options).finally(() => pendingCaptures.delete(page)); + void capture.catch(constVoid); + let timer: ReturnType | undefined; + const limit = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(noFrame()), CAPTURE_TIMEOUT_MS); + }); + return Promise.race([capture, limit]).finally(() => clearTimeout(timer)); +}; + export const snapshot = async (input: { readonly page: Page; readonly cdp: CDPSession; diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index b076a30a5221f..cc0a3258e7d88 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -15,6 +15,7 @@ import { previewEnvironment } from "~/state/preview"; import { useEnvironmentScope } from "~/state/session"; import { useAtomCommand } from "~/state/use-atom-command"; +import { holdFramesForCapture, releaseCaptureFrames } from "./agentFrameRequests"; import { readPreviewAnnotationTheme } from "./annotationTheme"; import { useBrowserDefaults } from "./browserDefaults"; import { useBrowserPointerStore } from "./browserPointerStore"; @@ -102,10 +103,24 @@ export function ElectronBrowserHost() { const openPreview = useAtomCommand(previewEnvironment.open, { reportFailure: true }); const sessionByRuntimeTabId = useRef(new Map()); useEffect(() => { - sessionByRuntimeTabId.current = new Map( - sessions.map((session) => [session.runtimeTabId, session]), - ); + const next = new Map(sessions.map((session) => [session.runtimeTabId, session])); + for (const tabId of sessionByRuntimeTabId.current.keys()) { + if (!next.has(tabId)) releaseCaptureFrames(tabId); + } + sessionByRuntimeTabId.current = next; }, [sessions]); + useEffect(() => { + const preview = window.desktopBridge?.preview; + if (!preview) return; + const unsubscribe = preview.onFrameRequest((tabId) => { + const session = sessionByRuntimeTabId.current.get(tabId); + if (session) holdFramesForCapture(tabId, session.threadRef); + }); + return () => { + unsubscribe(); + for (const tabId of sessionByRuntimeTabId.current.keys()) releaseCaptureFrames(tabId); + }; + }, []); useEffect(() => { const preview = window.desktopBridge?.preview; if (!preview) return; diff --git a/apps/web/src/browser/HostedBrowserWebview.test.tsx b/apps/web/src/browser/HostedBrowserWebview.test.tsx index aa582251cb079..6ba86087e7539 100644 --- a/apps/web/src/browser/HostedBrowserWebview.test.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.test.tsx @@ -99,6 +99,51 @@ afterEach(async () => { }); describe("HostedBrowserWebview settings hydration", () => { + it("releases a removed native guest without registering it when tab creation finishes late", async () => { + mocks.getClientSettings.mockResolvedValue(DEFAULT_CLIENT_SETTINGS); + await act(async () => ensureClientSettingsHydrated()); + const tabCreation = deferred(); + mocks.createTab.mockReturnValueOnce(tabCreation.promise); + const threadRef = { + environmentId: EnvironmentId.make("headless-fallback"), + threadId: ThreadId.make("thread-fallback"), + }; + const runtimeTabId = "late-native-fallback"; + await act(() => { + renderer = create( + , + { + createNodeMock: (element) => + element.type === "webview" + ? Object.assign(new EventTarget(), { getWebContentsId: () => 41 }) + : { scrollLeft: 0, scrollTop: 0, scrollTo: () => undefined }, + }, + ); + }); + expect(mocks.createTab).toHaveBeenCalledOnce(); + expect(mocks.registerWebview).not.toHaveBeenCalled(); + vi.useFakeTimers(); + await act(() => renderer?.unmount()); + renderer = undefined; + await act(async () => { + tabCreation.resolve(); + await tabCreation.promise; + await vi.advanceTimersByTimeAsync(0); + }); + expect(mocks.registerWebview).not.toHaveBeenCalled(); + expect(mocks.closeTab).toHaveBeenCalledExactlyOnceWith(runtimeTabId); + }); + it("starts a retained background tab only after a settings read succeeds on retry", async () => { const firstRead = deferred(); const retryRead = deferred(); diff --git a/apps/web/src/browser/agentFrameRequests.test.ts b/apps/web/src/browser/agentFrameRequests.test.ts new file mode 100644 index 0000000000000..b14f5b0fd329d --- /dev/null +++ b/apps/web/src/browser/agentFrameRequests.test.ts @@ -0,0 +1,81 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { Atom } from "effect/reactivity"; +import { appAtomRegistry } from "~/rpc/atomRegistry"; + +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { holdFramesForCapture, releaseCaptureFrames } from "./agentFrameRequests"; +import { useBrowserSurfaceStore } from "./browserSurfaceStore"; + +const shell = vi.hoisted(() => ({ atom: null as unknown })); +vi.mock("~/state/threads", () => ({ + environmentThreadShells: { threadShellAtom: () => shell.atom }, +})); +type Shell = { readonly runtime: { readonly status: string } | null }; +const shellAtom = Atom.make(null).pipe(Atom.keepAlive); +shell.atom = shellAtom; +const threadRef = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), +}; +const drawing = (tabId: string) => useBrowserSurfaceStore.getState().activityByTabId[tabId] ?? 0; + +afterEach(() => { + releaseCaptureFrames("tab-1"); + vi.useRealTimers(); +}); + +describe("capture frame leases", () => { + it("keeps an active turn drawing until it ends, including after the capture limit", () => { + vi.useFakeTimers(); + appAtomRegistry.set(shellAtom, { runtime: { status: "running" } }); + holdFramesForCapture("tab-1", threadRef); + holdFramesForCapture("tab-1", threadRef); + vi.advanceTimersByTime(10_000); + expect(drawing("tab-1")).toBe(1); + appAtomRegistry.set(shellAtom, { runtime: { status: "idle" } }); + expect(drawing("tab-1")).toBe(0); + }); + + it("bounds the lease when the background thread shell is missing", () => { + vi.useFakeTimers(); + appAtomRegistry.set(shellAtom, null); + holdFramesForCapture("tab-1", threadRef); + expect(drawing("tab-1")).toBe(1); + vi.advanceTimersByTime(5_000); + expect(drawing("tab-1")).toBe(0); + }); + + it("draws without thread state and stops after the capture limit", () => { + vi.useFakeTimers(); + holdFramesForCapture("tab-1"); + expect(drawing("tab-1")).toBe(1); + vi.advanceTimersByTime(5_000); + expect(drawing("tab-1")).toBe(0); + }); + + it("renews one lease for repeated captures", () => { + vi.useFakeTimers(); + holdFramesForCapture("tab-1"); + vi.advanceTimersByTime(4_000); + holdFramesForCapture("tab-1"); + expect(drawing("tab-1")).toBe(1); + vi.advanceTimersByTime(4_000); + expect(drawing("tab-1")).toBe(1); + vi.advanceTimersByTime(1_000); + expect(drawing("tab-1")).toBe(0); + }); + + it("releases a removed guest without releasing another activity lease", () => { + vi.useFakeTimers(); + const releaseRecording = useBrowserSurfaceStore.getState().acquireActivity("tab-1"); + holdFramesForCapture("tab-1"); + expect(drawing("tab-1")).toBe(2); + releaseCaptureFrames("tab-1"); + releaseCaptureFrames("tab-1"); + vi.advanceTimersByTime(5_000); + expect(drawing("tab-1")).toBe(1); + releaseRecording(); + expect(drawing("tab-1")).toBe(0); + }); +}); diff --git a/apps/web/src/browser/agentFrameRequests.ts b/apps/web/src/browser/agentFrameRequests.ts new file mode 100644 index 0000000000000..215e5984df225 --- /dev/null +++ b/apps/web/src/browser/agentFrameRequests.ts @@ -0,0 +1,37 @@ +import { threadRuntimeIsActive } from "@t3tools/client-runtime/state/models"; +import type { ScopedThreadRef } from "@t3tools/contracts"; + +import { appAtomRegistry } from "~/rpc/atomRegistry"; +import { environmentThreadShells } from "~/state/threads"; + +import { acquireBrowserSurfaceActivity } from "./browserSurfaceStore"; + +// Matches the server's capture limit when the thread shell is not loaded. +const CAPTURE_FRAME_LEASE_MS = 5_000; +const leases = new Map void; unsubscribe: () => void }>(); + +/** Keeps a hidden page drawing through an active turn, or for one bounded capture. */ +export function holdFramesForCapture(tabId: string, threadRef?: ScopedThreadRef): void { + const previous = leases.get(tabId); + previous?.unsubscribe(); + const release = previous?.release ?? acquireBrowserSurfaceActivity(tabId); + const shellAtom = threadRef ? environmentThreadShells.threadShellAtom(threadRef) : null; + if (shellAtom && threadRuntimeIsActive(appAtomRegistry.get(shellAtom)?.runtime)) { + const unsubscribe = appAtomRegistry.subscribe(shellAtom, (shell) => { + if (!threadRuntimeIsActive(shell?.runtime)) releaseCaptureFrames(tabId); + }); + leases.set(tabId, { release, unsubscribe }); + } else { + const timer = setTimeout(() => releaseCaptureFrames(tabId), CAPTURE_FRAME_LEASE_MS); + leases.set(tabId, { release, unsubscribe: () => clearTimeout(timer) }); + } +} + +/** Releases a frame lease when its native guest or host goes away. */ +export function releaseCaptureFrames(tabId: string): void { + const lease = leases.get(tabId); + if (!lease) return; + leases.delete(tabId); + lease.unsubscribe(); + lease.release(); +} diff --git a/apps/web/src/browser/previewRuntime.test.ts b/apps/web/src/browser/previewRuntime.test.ts index 951b52e904687..9169e8216f3a5 100644 --- a/apps/web/src/browser/previewRuntime.test.ts +++ b/apps/web/src/browser/previewRuntime.test.ts @@ -7,7 +7,11 @@ const state = vi.hoisted(() => ({ serverBrowser: new Set(), })); -vi.mock("~/env", () => ({ isElectron: false })); +vi.mock("~/env", () => ({ + get isElectron() { + return state.desktop; + }, +})); vi.mock("~/previewStateStore", () => ({ isPreviewSupportedInRuntime: () => state.desktop })); vi.mock("~/rpc/atomRegistry", () => ({ appAtomRegistry: { get: () => state.primary } })); vi.mock("~/state/primaryEnvironment", () => ({ primaryEnvironmentIdAtom: {} })); @@ -16,7 +20,11 @@ vi.mock("~/state/entities", () => ({ useEnvironmentSupportsServerBrowser: () => false, })); -import { alternatePreviewRuntime, previewRuntimeFor } from "./previewRuntime"; +import { + alternatePreviewRuntime, + previewRuntimeFor, + rendersServerTabNatively, +} from "./previewRuntime"; const local = "local" as EnvironmentId; const remote = "remote" as EnvironmentId; @@ -58,3 +66,44 @@ describe("alternatePreviewRuntime", () => { expect(alternatePreviewRuntime(remote, null, true, { runtime: "server" })).toBeNull(); }); }); + +describe("rendersServerTabNatively", () => { + it("streams a local tab after headless fallback while retaining native and legacy tabs", () => { + state.desktop = true; + expect(rendersServerTabNatively(local, local, { runtime: "server" })).toBe(true); + expect( + rendersServerTabNatively(local, local, { + runtime: "server", + serverPageMode: "native", + }), + ).toBe(true); + expect( + rendersServerTabNatively(local, local, { + runtime: "server", + serverPageMode: "headless", + }), + ).toBe(false); + }); + + it("never hosts remote or non-server tabs as local server webviews", () => { + state.desktop = true; + expect( + rendersServerTabNatively(remote, local, { + runtime: "server", + serverPageMode: "native", + }), + ).toBe(false); + expect(rendersServerTabNatively(local, null, { runtime: "server" })).toBe(false); + expect(rendersServerTabNatively(local, local, { runtime: "desktop" })).toBe(false); + expect(rendersServerTabNatively(local, local, null)).toBe(false); + }); + + it("streams native server pages in a web client", () => { + expect( + rendersServerTabNatively(local, local, { + runtime: "server", + serverPageMode: "native", + }), + ).toBe(false); + }); +}); diff --git a/apps/web/src/browser/previewRuntime.ts b/apps/web/src/browser/previewRuntime.ts index 435eb07bbc3c5..1818e4720757f 100644 --- a/apps/web/src/browser/previewRuntime.ts +++ b/apps/web/src/browser/previewRuntime.ts @@ -56,16 +56,18 @@ export function usePreviewAvailable(environmentId: EnvironmentId | null): boolea /** * Whether this client draws a server tab with its own ``. The desktop * app renders tabs of the server it launched, which drives them over the - * desktop browser channel; every other client and environment streams them. + * desktop browser channel; headless tabs and every other client and environment + * stream them. */ export function rendersServerTabNatively( environmentId: EnvironmentId, primaryEnvironmentId: EnvironmentId | null, - snapshot: Pick | null | undefined, + snapshot: Pick | null | undefined, ): boolean { return ( isElectron && snapshot?.runtime === "server" && + snapshot.serverPageMode !== "headless" && primaryEnvironmentId !== null && environmentId === primaryEnvironmentId ); @@ -73,7 +75,7 @@ export function rendersServerTabNatively( export function useRendersServerTabNatively( environmentId: EnvironmentId, - snapshot: Pick | null | undefined, + snapshot: Pick | null | undefined, ): boolean { return rendersServerTabNatively(environmentId, useAtomValue(primaryEnvironmentIdAtom), snapshot); } diff --git a/apps/web/src/components/preview/PreviewView.test.tsx b/apps/web/src/components/preview/PreviewView.test.tsx index 09eb7ee8e1d50..7dbb3a8c3c18f 100644 --- a/apps/web/src/components/preview/PreviewView.test.tsx +++ b/apps/web/src/components/preview/PreviewView.test.tsx @@ -6,10 +6,19 @@ import { EnvironmentId, FILL_PREVIEW_VIEWPORT, ThreadId, + type PreviewSessionSnapshot, } from "@t3tools/contracts"; -import { act, createElement, Profiler } from "react"; +import { + act, + createElement, + Profiler, + useEffect, + useImperativeHandle, + type ComponentProps, +} from "react"; import { renderToStaticMarkup } from "react-dom/server"; import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import type { ServerBrowserSurface } from "~/browser/ServerBrowserSurface"; const mocks = vi.hoisted(() => ({ navigate: vi.fn(async (_tabId: string, _url: string): Promise => undefined), @@ -40,6 +49,62 @@ const mocks = vi.hoisted(() => ({ recordingTabIds: new Set(), recordingRuntimeTabId: null as string | null, recordVisitForThread: vi.fn(), + electron: false, + runtime: undefined as PreviewSessionSnapshot["runtime"], + serverPageMode: undefined as PreviewSessionSnapshot["serverPageMode"], + chromeLoading: false, + chromeInputDisabled: false, + nativeSurfaceMounted: vi.fn(), + nativeSurfaceUnmounted: vi.fn(), + streamMounted: vi.fn(), + streamUnmounted: vi.fn(), + streamNavigate: vi.fn(), +})); + +vi.mock("~/env", async (importOriginal) => ({ + ...(await importOriginal()), + get isElectron() { + return mocks.electron; + }, +})); +vi.mock("~/browser/previewRuntime", async (importOriginal) => { + const runtime = await importOriginal(); + return { + ...runtime, + useRendersServerTabNatively: ( + environmentId: EnvironmentId, + snapshot: Pick | null | undefined, + ) => + runtime.rendersServerTabNatively( + environmentId, + EnvironmentId.make("environment-1"), + snapshot, + ), + }; +}); +vi.mock("~/browser/ServerBrowserSurface", () => ({ + ServerBrowserSurface: (props: ComponentProps) => { + useImperativeHandle( + props.ref, + () => ({ + navigate: mocks.streamNavigate, + history: vi.fn(), + reload: vi.fn(), + viewport: vi.fn(), + canvas: () => null, + }), + [], + ); + useEffect(() => { + mocks.streamMounted(props.tabId); + props.onControl?.({ controller: "you", canOperate: true, generation: 1, dialog: null }); + props.onFirstFrame?.(); + return () => { + mocks.streamUnmounted(props.tabId); + }; + }, []); + return createElement("server-surface"); + }, })); const EMPTY_HISTORY: never[] = []; @@ -135,6 +200,8 @@ vi.mock("~/previewStateStore", () => ({ "tab-1": { threadId: "thread-1", tabId: "tab-1", + runtime: mocks.runtime, + serverPageMode: mocks.serverPageMode, navStatus: { _tag: "Success", url: "http://example.com/", @@ -232,6 +299,8 @@ vi.mock("./previewBridge", () => ({ vi.mock("./PreviewChromeRow", () => ({ PreviewChromeRow: (props: { onSubmit: (url: string) => void; + loading: boolean; + inputDisabled?: boolean; onPickElement?: () => void; onPictureInPicture?: () => void; pictureInPicture?: boolean; @@ -240,6 +309,8 @@ vi.mock("./PreviewChromeRow", () => ({ }; }) => { mocks.submittedUrl = props.onSubmit; + mocks.chromeLoading = props.loading; + mocks.chromeInputDisabled = props.inputDisabled ?? false; mocks.toggleAnnotation = props.onPickElement ?? null; mocks.togglePictureInPicture = props.onPictureInPicture ?? null; mocks.toggleNativePictureInPicture = @@ -266,7 +337,17 @@ vi.mock("./ZoomIndicator", () => ({ ZoomIndicator: () => null })); vi.mock("./AgentBrowserCursor", () => ({ AgentBrowserCursor: () => createElement("agent-cursor"), })); -vi.mock("~/browser/BrowserSurfaceSlot", () => ({ BrowserSurfaceSlot: () => null })); +vi.mock("~/browser/BrowserSurfaceSlot", () => ({ + BrowserSurfaceSlot: (props: { tabId: string }) => { + useEffect(() => { + mocks.nativeSurfaceMounted(props.tabId); + return () => { + mocks.nativeSurfaceUnmounted(props.tabId); + }; + }, [props.tabId]); + return null; + }, +})); vi.mock("./usePreviewSession", () => ({ usePreviewSession: vi.fn() })); import { PreviewView } from "./PreviewView"; @@ -372,6 +453,52 @@ describe("PreviewView navigation", () => { mocks.recordingTabIds = new Set(); mocks.recordingRuntimeTabId = null; mocks.recordVisitForThread.mockClear(); + mocks.electron = false; + mocks.runtime = undefined; + mocks.serverPageMode = undefined; + mocks.nativeSurfaceMounted.mockClear(); + mocks.nativeSurfaceUnmounted.mockClear(); + mocks.streamMounted.mockClear(); + mocks.streamUnmounted.mockClear(); + mocks.streamNavigate.mockClear(); + }); + + it("switches a native server tab to its stream and drops stale desktop state", async () => { + mocks.electron = true; + mocks.runtime = "server"; + mocks.serverPageMode = "native"; + mocks.loading = true; + const document = installTestDom(); + const { createRoot } = await import("react-dom/client"); + const container = document.createElement("div"); + const root = createRoot(container as unknown as Element); + const hasCursor = (node: TestNode): boolean => + node.nodeName === "AGENT-CURSOR" || node.childNodes.some(hasCursor); + const view = () => ; + try { + await act(() => root.render(view())); + expect(mocks.nativeSurfaceMounted).toHaveBeenCalledWith(TEST_RUNTIME_TAB_ID); + expect(mocks.streamMounted).not.toHaveBeenCalled(); + expect(mocks.chromeLoading).toBe(true); + expect(hasCursor(container)).toBe(true); + + mocks.serverPageMode = "headless"; + await act(() => root.render(view())); + expect(mocks.nativeSurfaceUnmounted).toHaveBeenCalledWith(TEST_RUNTIME_TAB_ID); + expect(mocks.streamMounted).toHaveBeenCalledTimes(1); + expect(mocks.streamMounted).toHaveBeenCalledWith("tab-1"); + expect(mocks.chromeLoading).toBe(false); + expect(mocks.chromeInputDisabled).toBe(false); + expect(mocks.toggleAnnotation).toBeNull(); + expect(hasCursor(container)).toBe(false); + + await act(async () => mocks.submittedUrl?.("localhost:3000/stream")); + expect(mocks.streamNavigate).toHaveBeenCalledWith("http://localhost:3000/stream"); + expect(mocks.navigate).not.toHaveBeenCalled(); + } finally { + await act(() => root.unmount()); + vi.unstubAllGlobals(); + } }); it("shows the cursor in a replacement browser while the old instance still records", async () => { diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index 161fc4c888c87..992391596f987 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -226,7 +226,8 @@ export function PreviewView({ const [serverControlledTabId, setServerControlledTabId] = useState(null); const serverInputDisabled = isServerTab && serverControlledTabId !== runtimeTabId; const [serverFrameTabId, setServerFrameTabId] = useState(null); - const desktopOverlay = tabId ? (previewState.desktopByTabId[tabId] ?? null) : null; + const desktopOverlay = + !isServerTab && tabId ? (previewState.desktopByTabId[tabId] ?? null) : null; const navStatus = snapshot?.navStatus ?? { _tag: "Idle" as const }; const url = navStatus._tag === "Idle" ? "" : navStatus.url; const loading = desktopOverlay?.loading ?? navStatus._tag === "Loading"; diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index b249c44afc108..d69f49e0a8fdd 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -125,12 +125,12 @@ function BrowserMiniPlayer({ threadRef, tabId, miniPlayer }: Props & { readonly const recording = recordingTabIds.has(runtimeTabId) || findActiveBrowserRecordingRuntimeTabId(threadRef, tabId) !== null; - const desktopOverlay = previewState.desktopByTabId[tabId] ?? null; const fittedSourceContent = useBrowserSurfaceStore( (state) => state.byTabId[runtimeTabId]?.fittedSourceContent ?? null, ); const nativeServerTab = useRendersServerTabNatively(threadRef.environmentId, snapshot); const serverTab = snapshot?.runtime === "server" && !nativeServerTab; + const desktopOverlay = serverTab ? null : (previewState.desktopByTabId[tabId] ?? null); const [streamViewport, setStreamViewport] = useState(null); const serverSurfaceRef = useRef(null); const serverPictureInPicture = @@ -140,7 +140,7 @@ function BrowserMiniPlayer({ threadRef, tabId, miniPlayer }: Props & { readonly ? streamViewport : resolvePreviewMiniPlayerSourceSize( snapshot?.viewport ?? FILL_PREVIEW_VIEWPORT, - fittedSourceContent, + serverTab ? null : fittedSourceContent, desktopOverlay?.zoomFactor ?? 1, ); diff --git a/apps/web/src/components/preview/previewFallback.test.tsx b/apps/web/src/components/preview/previewFallback.test.tsx new file mode 100644 index 0000000000000..177dc64975a94 --- /dev/null +++ b/apps/web/src/components/preview/previewFallback.test.tsx @@ -0,0 +1,227 @@ +import { scopedThreadKey } from "@t3tools/client-runtime/environment"; +import { EnvironmentId, ThreadId, type PreviewSessionSnapshot } from "@t3tools/contracts"; +import { + act, + createElement, + useEffect, + useImperativeHandle, + type ComponentProps, + type ReactNode, +} from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import type { HostedBrowserWebview } from "~/browser/HostedBrowserWebview"; +import type { ServerBrowserSurface } from "~/browser/ServerBrowserSurface"; + +const mocks = vi.hoisted(() => ({ + mode: "native" as "native" | "headless", + nativeMount: vi.fn(), + nativeUnmount: vi.fn(), + streamMount: vi.fn(), + streamUnmount: vi.fn(), + reportPreview: vi.fn(), + clearPreview: vi.fn(), + nativePopOut: vi.fn(), + nativeClosePopOut: vi.fn(), + serverPopOut: vi.fn(), +})); + +const threadRef = { + environmentId: EnvironmentId.make("local"), + threadId: ThreadId.make("thread-1"), +}; +function previewState() { + return { + serverEpoch: "server-1", + sessions: { + "tab-1": { + threadId: threadRef.threadId, + tabId: "tab-1", + runtime: "server", + serverPageMode: mocks.mode, + navStatus: { _tag: "Success", url: "https://example.com/", title: "Example" }, + viewport: { _tag: "freeform", width: 800, height: 600 }, + profileId: "default", + canGoBack: false, + canGoForward: false, + updatedAt: "2026-10-11T00:00:00.000Z", + } satisfies PreviewSessionSnapshot, + }, + desktopByTabId: { + "tab-1": { hasWebContents: true, zoomFactor: 2, pictureInPicture: true }, + }, + }; +} + +vi.mock("~/env", () => ({ isElectron: true })); +vi.mock("@effect/atom-react", () => ({ useAtomValue: () => threadRef.environmentId })); +vi.mock("~/state/primaryEnvironment", () => ({ primaryEnvironmentIdAtom: {} })); +vi.mock("~/state/entities", () => ({ + readEnvironmentSupportsServerBrowser: () => true, + useEnvironmentSupportsServerBrowser: () => true, +})); +vi.mock("~/previewStateStore", () => ({ + useThreadPreviewState: previewState, + useActivePreviewSessions: () => ({ [scopedThreadKey(threadRef)]: previewState() }), + isPreviewSupportedInRuntime: () => true, +})); +vi.mock("~/hooks/useSettings", () => ({ useClientSettingsHydrated: () => true })); +vi.mock("~/hooks/useTheme", () => ({ useTheme: () => ({ resolvedTheme: "light" }) })); +vi.mock("~/browser/browserDefaults", () => ({ + useBrowserDefaults: () => ({ profileId: "default" }), +})); +vi.mock("~/browser/browserPointerStore", () => ({ useBrowserPointerStore: {} })); +vi.mock("~/state/session", () => ({ useEnvironmentScope: () => true })); +vi.mock("~/state/preview", () => ({ previewEnvironment: { open: {} } })); +vi.mock("~/state/use-atom-command", () => ({ useAtomCommand: () => vi.fn() })); +vi.mock("~/browser/openFileInPreview", () => ({ openUrlInPreview: vi.fn() })); +vi.mock("~/browser/HostedBrowserWebview", () => ({ + HostedBrowserWebview: (props: ComponentProps) => { + useEffect(() => { + mocks.nativeMount(props.runtimeTabId); + return () => { + mocks.nativeUnmount(props.runtimeTabId); + }; + }, [props.runtimeTabId]); + return createElement("webview"); + }, +})); +vi.mock("~/browser/browserRecording", () => ({ + findActiveBrowserRecordingRuntimeTabId: () => null, + useActiveBrowserRecordingTabIds: () => new Set(), +})); +vi.mock("~/browser/browserSurfaceStore", () => ({ + useBrowserSurfaceStore: (select: (state: { byTabId: Record }) => unknown) => + select({ byTabId: {} }), +})); +vi.mock("~/browser/BrowserSurfaceSlot", () => ({ + BrowserSurfaceSlot: () => createElement("span", { "aria-label": "Native surface" }), +})); +vi.mock("~/browser/ServerBrowserSurface", () => ({ + ServerBrowserSurface: (props: ComponentProps) => { + useImperativeHandle( + props.ref, + () => ({ + navigate: vi.fn(), + history: vi.fn(), + reload: vi.fn(), + viewport: vi.fn(), + canvas: () => null, + }), + [], + ); + useEffect(() => { + mocks.streamMount(props.tabId); + return () => { + mocks.streamUnmount(props.tabId); + }; + }, [props.tabId]); + return createElement("stream-surface"); + }, +})); +vi.mock("~/browser/serverPictureInPicture", () => ({ + closeServerPictureInPicture: vi.fn(), + openServerPictureInPicture: mocks.serverPopOut, + serverPictureInPictureKey: (threadId: string, tabId: string) => `${threadId}:${tabId}`, + supportsServerPictureInPicture: () => true, + useServerPictureInPictureKey: () => null, +})); +vi.mock("./previewBridge", () => ({ + previewBridge: { pictureInPicture: { open: mocks.nativePopOut, close: mocks.nativeClosePopOut } }, +})); +vi.mock("../chat/ChatCanvasContext", () => ({ + useChatCanvas: () => ({ + previewKey: "browser:tab-1", + layout: { frame: { x: 0, y: 0, width: 240, height: 180 } }, + reportPreview: mocks.reportPreview, + clearPreview: mocks.clearPreview, + }), +})); +vi.mock("~/state/device", () => ({ + useDeviceState: () => ({ state: { devices: [], hosts: [] } }), +})); +vi.mock("../device/DeviceStreamView", () => ({ DeviceStreamView: () => null })); +vi.mock("~/components/ui/button", () => ({ + Button: (props: ComponentProps<"button">) => createElement("button", props), +})); +vi.mock("~/components/ui/tooltip", () => ({ + Tooltip: (props: { children: ReactNode }) => props.children, + TooltipTrigger: (props: { render: ReactNode }) => props.render, + TooltipPopup: () => null, +})); + +import { ElectronBrowserHost } from "~/browser/ElectronBrowserHost"; +import { ThreadPreviewMiniPlayer } from "./ThreadPreviewMiniPlayer"; +import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId"; + +let renderer: ReactTestRenderer | undefined; +beforeEach(() => { + vi.clearAllMocks(); + mocks.mode = "native"; + vi.stubGlobal("window", {}); + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); +}); +afterEach(async () => { + await act(() => renderer?.unmount()); + renderer = undefined; + vi.unstubAllGlobals(); +}); + +describe("headless fallback surfaces", () => { + it("unmounts the hosted native guest and never remounts it for a headless snapshot", async () => { + const runtimeTabId = previewRuntimeTabId(threadRef, "server-1", "tab-1"); + await act(() => { + renderer = create(); + }); + expect(mocks.nativeMount).toHaveBeenCalledExactlyOnceWith(runtimeTabId); + expect(renderer!.root.findAllByType("webview")).toHaveLength(1); + mocks.mode = "headless"; + await act(() => renderer!.update()); + expect(mocks.nativeUnmount).toHaveBeenCalledExactlyOnceWith(runtimeTabId); + expect(renderer!.root.findAllByType("webview")).toHaveLength(0); + await act(() => renderer!.update()); + expect(mocks.nativeMount).toHaveBeenCalledTimes(1); + }); + + it("moves a floating tab to its stream and uses server pop-out instead of stale native PiP", async () => { + const view = () => ( + + ); + await act(() => { + renderer = create(view()); + }); + expect(renderer!.root.findAllByProps({ "aria-label": "Native surface" })).toHaveLength(1); + const nativeCloseButton = renderer!.root.findByProps({ + "aria-label": "Close popped-out preview", + }); + await act(async () => nativeCloseButton.props.onClick()); + expect(mocks.nativeClosePopOut).toHaveBeenCalledOnce(); + + mocks.mode = "headless"; + await act(() => renderer!.update(view())); + expect(renderer!.root.findAllByProps({ "aria-label": "Native surface" })).toHaveLength(0); + expect(mocks.streamMount).toHaveBeenCalledExactlyOnceWith("tab-1"); + expect(mocks.reportPreview).toHaveBeenLastCalledWith( + expect.objectContaining({ source: { width: 800, height: 600 } }), + ); + const serverOpenButton = renderer!.root.findByProps({ + "aria-label": "Pop preview into separate window", + }); + await act(async () => serverOpenButton.props.onClick()); + expect(mocks.serverPopOut).toHaveBeenCalledExactlyOnceWith({ + ...threadRef, + tabId: "tab-1", + seed: null, + }); + expect(mocks.nativeClosePopOut).toHaveBeenCalledOnce(); + expect(mocks.nativePopOut).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/components/preview/usePreviewBridge.test.ts b/apps/web/src/components/preview/usePreviewBridge.test.ts index 14acf0d69609e..31fe7863c5bca 100644 --- a/apps/web/src/components/preview/usePreviewBridge.test.ts +++ b/apps/web/src/components/preview/usePreviewBridge.test.ts @@ -1,7 +1,69 @@ -import type { DesktopPreviewTabState } from "@t3tools/contracts"; -import { describe, expect, it } from "vite-plus/test"; +import { EnvironmentId, ThreadId, type DesktopPreviewTabState } from "@t3tools/contracts"; +import * as Option from "effect/Option"; +import { act, createElement } from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -import { projectDesktopState } from "./usePreviewBridge"; +const mocks = vi.hoisted(() => ({ + mode: undefined as "native" | "headless" | undefined, + listener: null as ((tabId: string, state: DesktopPreviewTabState) => void) | null, + applyDesktopState: vi.fn(), + unsubscribe: vi.fn(), + reportStatus: vi.fn(), + clearPointer: vi.fn(), +})); + +vi.mock("~/previewStateStore", () => ({ + applyPreviewDesktopState: mocks.applyDesktopState, + readThreadPreviewState: () => ({ sessions: { "tab-1": { serverPageMode: mocks.mode } } }), +})); +vi.mock("~/browserFaviconStore", () => ({ + flushPendingFaviconsForThread: vi.fn(), + recordFaviconForThread: vi.fn(), + useFaviconProjectRefForThread: () => null, +})); +vi.mock("~/browser/browserPointerStore", () => ({ + useBrowserPointerStore: (select: (state: { clear: typeof mocks.clearPointer }) => unknown) => + select({ clear: mocks.clearPointer }), +})); +vi.mock("~/state/preview", () => ({ previewEnvironment: { reportStatus: {} } })); +vi.mock("~/state/session", () => ({ usePreparedConnection: () => Option.none() })); +vi.mock("~/state/use-atom-command", () => ({ useAtomCommand: () => mocks.reportStatus })); +vi.mock("./previewBridge", () => ({ + previewBridge: { + onStateChange: (listener: typeof mocks.listener) => { + mocks.listener = listener; + return mocks.unsubscribe; + }, + }, +})); + +import { projectDesktopState, usePreviewBridge } from "./usePreviewBridge"; + +const threadRef = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), +}; +let renderer: ReactTestRenderer | undefined; + +function BridgeHarness() { + usePreviewBridge({ threadRef, tabId: "tab-1", runtimeTabId: "native-tab-1", serverDriven: true }); + return null; +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.mode = undefined; + mocks.listener = null; + vi.stubGlobal("window", {}); + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); +}); + +afterEach(async () => { + await act(() => renderer?.unmount()); + renderer = undefined; + vi.unstubAllGlobals(); +}); const favicon = { dataUrl: "data:image/png;base64,AAAA", @@ -48,3 +110,50 @@ describe("projectDesktopState", () => { expect(projectDesktopState(state({ kind: "Idle" })).favicon).toBeNull(); }); }); + +describe("native bridge lifetime", () => { + it("drops late native state when the server switches this tab to headless", async () => { + mocks.mode = "native"; + await act(() => { + renderer = create(createElement(BridgeHarness)); + }); + mocks.listener?.( + "native-tab-1", + state({ kind: "Loading", url: "https://example.com/", title: "" }), + ); + expect(mocks.applyDesktopState).toHaveBeenLastCalledWith( + threadRef, + "tab-1", + expect.objectContaining({ loading: true }), + ); + expect(mocks.reportStatus).not.toHaveBeenCalled(); + + mocks.mode = "headless"; + mocks.listener?.( + "native-tab-1", + state({ kind: "Success", url: "https://other.example/", title: "Late" }), + ); + expect(mocks.applyDesktopState).toHaveBeenLastCalledWith(threadRef, "tab-1", null); + expect(mocks.reportStatus).not.toHaveBeenCalled(); + + await act(() => renderer?.unmount()); + renderer = undefined; + expect(mocks.unsubscribe).toHaveBeenCalledTimes(1); + expect(mocks.applyDesktopState).toHaveBeenLastCalledWith(threadRef, "tab-1", null); + }); + + it("clears the native overlay on headless unmount even without another native event", async () => { + await act(() => { + renderer = create(createElement(BridgeHarness)); + }); + mocks.listener?.( + "native-tab-1", + state({ kind: "Loading", url: "https://example.com/", title: "" }), + ); + mocks.mode = "headless"; + await act(() => renderer?.unmount()); + renderer = undefined; + expect(mocks.unsubscribe).toHaveBeenCalledTimes(1); + expect(mocks.applyDesktopState).toHaveBeenLastCalledWith(threadRef, "tab-1", null); + }); +}); diff --git a/apps/web/src/components/preview/usePreviewBridge.ts b/apps/web/src/components/preview/usePreviewBridge.ts index ee737892a58d0..83772b35b3b97 100644 --- a/apps/web/src/components/preview/usePreviewBridge.ts +++ b/apps/web/src/components/preview/usePreviewBridge.ts @@ -16,7 +16,11 @@ import { useFaviconProjectRefForThread, } from "~/browserFaviconStore"; import { useBrowserPointerStore } from "~/browser/browserPointerStore"; -import { applyPreviewDesktopState, type DesktopPreviewOverlay } from "~/previewStateStore"; +import { + applyPreviewDesktopState, + readThreadPreviewState, + type DesktopPreviewOverlay, +} from "~/previewStateStore"; import { previewEnvironment } from "~/state/preview"; import { usePreparedConnection } from "~/state/session"; import { useAtomCommand } from "~/state/use-atom-command"; @@ -66,6 +70,11 @@ export function usePreviewBridge(input: { const handleStateChange = useEffectEvent( (changedTabId: string, state: DesktopPreviewTabState): void => { if (changedTabId !== runtimeTabId) return; + // A late native guest must not override the headless page's stream state. + if (readThreadPreviewState(stableThreadRef).sessions[tabId]?.serverPageMode === "headless") { + applyPreviewDesktopState(stableThreadRef, tabId, null); + return; + } if (shouldClearBrowserPointer(lastDesktopNavStatus.current, state.navStatus)) { clearBrowserPointer(runtimeTabId); } @@ -95,7 +104,13 @@ export function usePreviewBridge(input: { lastReportedUrl.current = null; lastReportedKind.current = null; lastDesktopNavStatus.current = null; - return bridge.onStateChange(handleStateChange); + const unsubscribe = bridge.onStateChange(handleStateChange); + return () => { + unsubscribe(); + if (readThreadPreviewState(stableThreadRef).sessions[tabId]?.serverPageMode === "headless") { + applyPreviewDesktopState(stableThreadRef, tabId, null); + } + }; }, [bridge, runtimeTabId, stableThreadRef, tabId]); useEffect(() => { if (!projectRef) return; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index e521c4c37271c..1a8645061af1e 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1352,6 +1352,8 @@ export interface DesktopPreviewBridge { }; onStateChange: (listener: (tabId: string, state: DesktopPreviewTabState) => void) => () => void; onPointerEvent: (listener: (event: DesktopPreviewPointerEvent) => void) => () => void; + /** The server waits for a frame of this hidden tab; keep it drawing for the capture limit. */ + onFrameRequest: (listener: (tabId: string) => void) => () => void; onOpenLink: (listener: (event: DesktopPreviewOpenLinkEvent) => void) => () => void; } diff --git a/packages/contracts/src/preview.test.ts b/packages/contracts/src/preview.test.ts index 24f429745ef87..707ef3d5f777a 100644 --- a/packages/contracts/src/preview.test.ts +++ b/packages/contracts/src/preview.test.ts @@ -80,21 +80,45 @@ describe("PreviewNavStatus", () => { }); describe("PreviewSessionSnapshot", () => { - it("round-trips a Success snapshot", () => { + const legacySnapshot = { + threadId: "thread-1", + tabId: "preview-thread-1", + navStatus: { + _tag: "Success" as const, + url: "http://localhost:5173/", + title: "Vite App", + }, + canGoBack: false, + canGoForward: false, + updatedAt: "2026-01-01T00:00:00.000Z", + }; + const jsonCodec = Schema.toCodecJson(PreviewSessionSnapshot); + const encodeJson = Schema.encodeSync(jsonCodec); + const decodeJson = Schema.decodeUnknownSync(jsonCodec); + + it("round-trips a legacy snapshot without choosing a page backing", () => { + const snapshot = decodeJson(encodeJson(decodeSnapshot(legacySnapshot))); + expect(snapshot).toEqual(legacySnapshot); + expect(snapshot).not.toHaveProperty("serverPageMode"); + }); + + it.each(["native", "headless"] as const)("preserves %s server page backing over JSON", (mode) => { const snapshot = decodeSnapshot({ - threadId: "thread-1", - tabId: "preview-thread-1", - navStatus: { - _tag: "Success", - url: "http://localhost:5173/", - title: "Vite App", - }, - canGoBack: false, - canGoForward: false, - updatedAt: "2026-01-01T00:00:00.000Z", + ...legacySnapshot, + runtime: "server", + serverPageMode: mode, + }); + expect(decodeJson(encodeJson(snapshot))).toEqual({ + ...legacySnapshot, + runtime: "server", + serverPageMode: mode, }); - expect(snapshot.tabId).toBe("preview-thread-1"); - expect(snapshot.navStatus._tag).toBe("Success"); + }); + + it("rejects a runtime value used as a page backing", () => { + expect(() => + decodeSnapshot({ ...legacySnapshot, runtime: "server", serverPageMode: "server" }), + ).toThrow(); }); }); @@ -244,6 +268,31 @@ describe("PreviewEvent", () => { expect(event.type).toBe("opened"); }); + it("preserves the headless fallback in a snapshot update sent to clients", () => { + const event = decodePreviewEvent({ + type: "navigated", + threadId: "t", + tabId: "preview-t", + createdAt: "2026-01-01T00:00:00.000Z", + serverEpoch: "server-a", + revision: 2, + snapshot: { + threadId: "t", + tabId: "preview-t", + runtime: "server", + serverPageMode: "headless", + navStatus: { _tag: "Loading", url: "https://example.com/", title: "" }, + canGoBack: false, + canGoForward: false, + updatedAt: "2026-01-01T00:00:00.000Z", + }, + }); + const jsonCodec = Schema.toCodecJson(PreviewEvent); + const received = Schema.decodeUnknownSync(jsonCodec)(Schema.encodeSync(jsonCodec)(event)); + expect(received).toEqual(event); + expect(received.type === "navigated" && received.snapshot.serverPageMode).toBe("headless"); + }); + it("decodes failed with code/description", () => { const event = decodePreviewEvent({ type: "failed", diff --git a/packages/contracts/src/preview.ts b/packages/contracts/src/preview.ts index 802bbd4765b7c..2d61108b0fb9a 100644 --- a/packages/contracts/src/preview.ts +++ b/packages/contracts/src/preview.ts @@ -172,10 +172,10 @@ export const PreviewNavStatus = Schema.Union([ export type PreviewNavStatus = typeof PreviewNavStatus.Type; /** - * Where a tab's page runs. `desktop` is an Electron owned by one - * desktop client; `server` is headless Chromium owned by the environment - * server, viewed by any client through `/api/preview-stream` and driven by - * agents with no client attached. Absent means `desktop`. + * Who manages a tab. `desktop` is an Electron owned by one desktop + * client; `server` is owned by the environment server and driven by agents. + * A server tab can use its desktop host's native page or headless Chromium; + * other clients view either through `/api/preview-stream`. Absent means `desktop`. */ export const PreviewRuntime = Schema.Literals(["desktop", "server"]); export type PreviewRuntime = typeof PreviewRuntime.Type; @@ -213,6 +213,8 @@ export const PreviewSessionSnapshot = Schema.Struct({ */ profileId: Schema.optional(BrowserProfileId), runtime: Schema.optional(PreviewRuntime), + /** Server-selected page backing. Absent preserves clients' existing host-based rendering choice. */ + serverPageMode: Schema.optional(Schema.Literals(["native", "headless"])), /** Authenticated provider session owning an isolated server tab. */ automationOwner: Schema.optional(Schema.String), /** An agent opened this tab and asked to show it, so viewers float it. */ diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 504d28148b296..410e5e8014379 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -92,6 +92,8 @@ export const PreviewAutomationStatus = Schema.Struct({ url: Schema.NullOr(Schema.String), title: Schema.NullOr(Schema.String), loading: Schema.Boolean, + /** Actual server page backing. Older hosts may omit it. */ + serverPageMode: Schema.optional(Schema.Literals(["native", "headless"])), control: Schema.optional( Schema.Struct({ owner: PreviewAutomationTabOwner, @@ -143,6 +145,7 @@ export const PreviewAutomationStatus = Schema.Struct({ ownedByCaller: Schema.optional(Schema.Boolean), visible: Schema.optional(Schema.Boolean), profileId: Schema.optional(BrowserProfileId), + serverPageMode: Schema.optional(Schema.Literals(["native", "headless"])), }), ), ),