diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 99df18c10cf5a..1b910bf862766 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -14,8 +14,10 @@ import { ThreadId, type PreviewAutomationSnapshot, type PreviewAutomationStatus, + type PreviewEvent, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -23,6 +25,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; +import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import type { BrowserContext, Page } from "playwright-core"; import { beforeEach, expect, vi } from "vite-plus/test"; @@ -197,6 +200,9 @@ function makeContext(onClose?: (context: BrowserContext) => void) { const context = { page, sessions, + setPageUrl: (next: string) => { + url = next; + }, newPage: async () => page as unknown as Page, grantPermissions: vi.fn(async () => {}), newCDPSession: async (target: unknown) => { @@ -359,14 +365,41 @@ const ready = Effect.gen(function* () { const tabId = PreviewTabId.make(opened.tabId!); return { browser, broker, tabId }; }); -const viewerInput = (tabId: string, canOperate: boolean) => ({ +const viewerInput = ( + tabId: string, + canOperate: boolean, + sessionId?: string, + handoffId?: string, +) => ({ threadId: scope.thread.threadId, tabId, canOperate, + ...(sessionId === undefined ? {} : { sessionId }), + ...(handoffId === undefined ? {} : { handoffId }), maxWidth: 1280, maxHeight: 800, quality: 70, }); +const nextLoaded = (events: PubSub.Subscription, tabId: string) => + Effect.gen(function* () { + while (true) { + const next = yield* PubSub.take(events); + if ( + next.type === "navigated" && + next.tabId === tabId && + next.snapshot.navStatus._tag === "Success" + ) { + return next.snapshot.navStatus; + } + } + }); +const nextPopup = (output: Queue.Dequeue) => + Effect.gen(function* () { + while (true) { + const next = yield* Queue.take(output); + if (next._tag === "popup") return next; + } + }); beforeEach(() => { contexts.length = 0; @@ -687,6 +720,51 @@ it.live("streams to a read-only viewer without allowing takeover, input, or view ).pipe(Effect.provide(layer)), ); +it.live("an adopted PDF popup reports success only after its recovery reload loads", () => + Effect.scoped( + Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const manager = yield* Manager.PreviewManager; + const events = yield* manager.subscribeEvents; + const reportedEvents = yield* manager.subscribeEvents; + const opened = yield* manager.open({ threadId: scope.thread.threadId, runtime: "server" }); + const viewer = yield* browser.attachViewer(viewerInput(opened.tabId, true)); + yield* viewer.input({ type: "takeControl" }); + const popup = makeContext(); + const url = "https://environment.test/api/assets/token/guide.pdf"; + yield* Effect.promise(() => popup.page.goto(url)); + popup.page.goto.mockClear(); + const reload = Promise.withResolvers(); + popup.page.reload.mockImplementationOnce(() => reload.promise); + contexts[0]!.page.emit("popup", popup.page); + let shown = yield* Queue.take(viewer.output); + while (shown._tag !== "popup") shown = yield* Queue.take(viewer.output); + + yield* browser.attachViewer(viewerInput(shown.tabId, true)); + expect(popup.page.reload).toHaveBeenCalledTimes(1); + expect(popup.page.title).not.toHaveBeenCalled(); + + popup.page.emit("load"); + reload.resolve(); + expect(yield* nextLoaded(events, shown.tabId)).toEqual({ + _tag: "Success", + url, + title: "test page", + }); + expect(popup.page.title).toHaveBeenCalledTimes(1); + const successes = yield* PubSub.takeUpTo(reportedEvents, 100); + expect( + successes.filter( + (event) => + event.type === "navigated" && + event.tabId === shown.tabId && + event.snapshot.navStatus._tag === "Success", + ), + ).toHaveLength(1); + }), + ).pipe(Effect.provide(layer)), +); + it.live( "takeover waits for the running agent and revokes snapshot refs before returning control", () => @@ -827,13 +905,116 @@ it.live("reports a pending dialog without evaluating the page and resolves it ex ).pipe(Effect.provide(layer)), ); +it.live( + "an adopted popup that finished loading before listeners attach reports its loaded page", + () => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const events = yield* manager.subscribeEvents; + const opener = contexts[0]!.page; + const popup = makeContext(); + const url = "https://example.com/popup"; + yield* Effect.promise(() => popup.page.goto(url)); + + opener.emit("popup", popup.page); + let opened = yield* PubSub.take(events); + while (opened.type !== "opened" || opened.snapshot.tabId === tabId) { + opened = yield* PubSub.take(events); + } + + yield* browser.attachViewer(viewerInput(opened.snapshot.tabId, false)); + expect(yield* nextLoaded(events, opened.snapshot.tabId)).toEqual({ + _tag: "Success", + url, + title: "test page", + }); + expect(popup.page.goto).toHaveBeenCalledTimes(1); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("an adopted popup that is still loading is not reported as successful early", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const events = yield* manager.subscribeEvents; + const opener = contexts[0]!.page; + const popup = makeContext(); + const url = "https://example.com/popup-loading"; + const loadState = Promise.withResolvers(); + popup.setPageUrl(url); + popup.page.waitForLoadState.mockImplementationOnce(() => loadState.promise); + + opener.emit("popup", popup.page); + let opened = yield* PubSub.take(events); + while (opened.type !== "opened" || opened.snapshot.tabId === tabId) { + opened = yield* PubSub.take(events); + } + + yield* browser.attachViewer(viewerInput(opened.snapshot.tabId, false)); + popup.page.emit("framenavigated", popup.page); + expect(popup.page.waitForLoadState).toHaveBeenCalledWith("load"); + // reportLoaded reads the title first, so an early report would have called it by now. + expect(popup.page.title).not.toHaveBeenCalled(); + expect( + (yield* manager.list({ threadId: scope.thread.threadId })).sessions.find( + (session) => session.tabId === opened.snapshot.tabId, + )?.navStatus, + ).toEqual({ _tag: "Loading", url, title: "" }); + + loadState.resolve(); + expect(yield* nextLoaded(events, opened.snapshot.tabId)).toEqual({ + _tag: "Success", + url, + title: "test page", + }); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("an adopted popup that loads after setup reports success once", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const events = yield* manager.subscribeEvents; + const opener = contexts[0]!.page; + const popup = makeContext(); + const url = "https://example.com/popup-loads-later"; + const loadState = Promise.withResolvers(); + popup.setPageUrl(url); + popup.page.waitForLoadState.mockImplementationOnce(() => loadState.promise); + + opener.emit("popup", popup.page); + let opened = yield* PubSub.take(events); + while (opened.type !== "opened" || opened.snapshot.tabId === tabId) { + opened = yield* PubSub.take(events); + } + yield* browser.attachViewer(viewerInput(opened.snapshot.tabId, false)); + + // Playwright settles the load-state wait and emits `load` for the same load. + loadState.resolve(); + popup.page.emit("load"); + expect(yield* nextLoaded(events, opened.snapshot.tabId)).toEqual({ + _tag: "Success", + url, + title: "test page", + }); + // Each report reads the title first, and a duplicate starts before the first one publishes. + expect(popup.page.title).toHaveBeenCalledTimes(1); + }), + ).pipe(Effect.provide(layer)), +); + it.live("a tab opened to a file the browser cannot show reports the file to download", () => Effect.scoped( Effect.gen(function* () { const browser = yield* ServerBrowser.ServerBrowser; const manager = yield* Manager.PreviewManager; const events = yield* manager.subscribeEvents; - yield* Effect.yieldNow; const opened = yield* manager.open({ threadId: scope.thread.threadId, runtime: "server" }); const viewer = yield* browser.attachViewer(viewerInput(opened.tabId, true)); const page = contexts[0]!.page; @@ -919,6 +1100,376 @@ it.live( ).pipe(Effect.provide(layer)), ); +it.live("a popup a person opens from their own tab reports its page once loaded", () => + Effect.scoped( + Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const manager = yield* Manager.PreviewManager; + yield* Effect.yieldNow; + const opened = yield* manager.open({ threadId: scope.thread.threadId, runtime: "server" }); + const viewer = yield* browser.attachViewer(viewerInput(opened.tabId, true)); + yield* viewer.input({ type: "takeControl" }); + const events = yield* manager.subscribeEvents; + const opener = contexts[0]!.page; + const popup = makeContext(); + const url = "https://example.com/sign-in"; + yield* Effect.promise(() => popup.page.goto(url)); + + opener.emit("popup", popup.page); + let shown = yield* Queue.take(viewer.output); + while (shown._tag !== "popup") shown = yield* Queue.take(viewer.output); + yield* browser.attachViewer(viewerInput(shown.tabId, true)); + expect(yield* nextLoaded(events, shown.tabId)).toEqual({ + _tag: "Success", + url, + title: "test page", + }); + expect(popup.page.goto).toHaveBeenCalledTimes(1); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("the person who opens a popup from an agent tab controls the popup", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, broker, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const sessionId = "opener-session"; + const watcher = yield* browser.attachViewer(viewerInput(tabId, false)); + const viewer = yield* browser.attachViewer(viewerInput(tabId, true, sessionId)); + yield* viewer.input({ type: "takeControl" }); + yield* Queue.clear(viewer.output); + const popup = makeContext(); + const waitingForPopup = yield* nextPopup(viewer.output).pipe(Effect.forkScoped); + const events = yield* manager.subscribeEvents; + const opener = contexts[0]!.page; + opener.emit("popup", popup.page); + let opened = yield* PubSub.take(events); + while (opened.type !== "opened") opened = yield* PubSub.take(events); + expect(opened.snapshot.reveal).toBeUndefined(); + const shown = yield* Fiber.join(waitingForPopup); + const popupTabId = PreviewTabId.make(shown.tabId); + const openerTab = (yield* manager.list({ threadId: scope.thread.threadId })).sessions.find( + (session) => session.tabId === tabId, + ); + expect(openerTab?.automationOwner).toBeDefined(); + expect(opened.snapshot).toMatchObject({ + tabId: popupTabId, + automationOwner: openerTab?.automationOwner, + }); + expect(opened.snapshot.reveal).toBeUndefined(); + + const popupViewer = yield* browser.attachViewer( + viewerInput(popupTabId, true, sessionId, shown.handoffId), + ); + let control = yield* Queue.take(popupViewer.output); + while (control._tag !== "control") control = yield* Queue.take(popupViewer.output); + expect(control).toMatchObject({ controller: "you" }); + + popup.page.openFileChooser(); + let offered = yield* Queue.take(popupViewer.output); + while (offered._tag !== "fileChooser") offered = yield* Queue.take(popupViewer.output); + expect(popup.page.nativeFileChooser).not.toHaveBeenCalled(); + + const refused = yield* broker + .invoke({ + scope, + tabId: popupTabId, + operation: "evaluate", + input: { expression: "read()" }, + }) + .pipe(Effect.flip); + expect(refused).toMatchObject({ reason: "humanControl" }); + const sessions = (yield* manager.list({ threadId: scope.thread.threadId })).sessions; + expect(sessions.find((session) => session.tabId === popupTabId)?.automationOwner).toBe( + sessions.find((session) => session.tabId === tabId)?.automationOwner, + ); + expect((yield* Queue.clear(watcher.output)).some((item) => item._tag === "popup")).toBe( + false, + ); + expect(opener.close).not.toHaveBeenCalled(); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("a watcher from the handoff session does not consume the popup handoff", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const sessionId = "opener-session"; + const opener = yield* browser.attachViewer(viewerInput(tabId, true, sessionId)); + yield* opener.input({ type: "takeControl" }); + yield* Queue.clear(opener.output); + const waitingForPopup = yield* nextPopup(opener.output).pipe(Effect.forkScoped); + contexts[0]!.page.emit("popup", makeContext().page); + const shown = yield* Fiber.join(waitingForPopup); + + const watcher = yield* browser.attachViewer(viewerInput(shown.tabId, false, sessionId)); + let control = yield* Queue.take(watcher.output); + while (control._tag !== "control") control = yield* Queue.take(watcher.output); + expect(control).toMatchObject({ controller: "agent" }); + + const sameSession = yield* browser.attachViewer(viewerInput(shown.tabId, true, sessionId)); + control = yield* Queue.take(sameSession.output); + while (control._tag !== "control") control = yield* Queue.take(sameSession.output); + expect(control).toMatchObject({ controller: "agent" }); + + const otherSession = yield* browser.attachViewer( + viewerInput(shown.tabId, true, "other-session"), + ); + control = yield* Queue.take(otherSession.output); + while (control._tag !== "control") control = yield* Queue.take(otherSession.output); + expect(control).toMatchObject({ controller: "agent" }); + + const operator = yield* browser.attachViewer( + viewerInput(shown.tabId, true, sessionId, shown.handoffId), + ); + control = yield* Queue.take(operator.output); + while (control._tag !== "control") control = yield* Queue.take(operator.output); + expect(control).toMatchObject({ controller: "you" }); + + yield* operator.input({ type: "releaseControl" }); + const later = yield* browser.attachViewer( + viewerInput(shown.tabId, true, sessionId, shown.handoffId), + ); + control = yield* Queue.take(later.output); + while (control._tag !== "control") control = yield* Queue.take(later.output); + expect(control).toMatchObject({ controller: "agent" }); + }), + ).pipe(Effect.provide(layer)), +); + +it.live( + "a clicker who releases the opener lets the agent keep the popup (e.g. saved-password login)", + () => + Effect.scoped( + Effect.gen(function* () { + const { browser, broker, tabId } = yield* ready; + const sessionId = "opener-session"; + const opener = yield* browser.attachViewer(viewerInput(tabId, true, sessionId)); + yield* opener.input({ type: "takeControl" }); + yield* Queue.clear(opener.output); + const waitingForPopup = yield* nextPopup(opener.output).pipe(Effect.forkScoped); + contexts[0]!.page.emit("popup", makeContext().page); + const shown = yield* Fiber.join(waitingForPopup); + + yield* opener.input({ type: "releaseControl" }); + const popupViewer = yield* browser.attachViewer( + viewerInput(shown.tabId, true, sessionId, shown.handoffId), + ); + let control = yield* Queue.take(popupViewer.output); + while (control._tag !== "control") control = yield* Queue.take(popupViewer.output); + expect(control).toMatchObject({ controller: "agent" }); + + expect( + yield* broker.invoke({ + scope, + tabId: shown.tabId, + operation: "evaluate", + input: { expression: "savedPasswordLogin()" }, + }), + ).toBe("evaluated"); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("releasing the opener while its popup is still starting lets the agent keep it", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const sessionId = "opener-session"; + const opener = yield* browser.attachViewer(viewerInput(tabId, true, sessionId)); + yield* opener.input({ type: "takeControl" }); + yield* Queue.clear(opener.output); + const popup = makeContext(); + const settingUp = Promise.withResolvers(); + const setUp = Promise.withResolvers(); + popup.page.setViewportSize.mockImplementationOnce(() => { + settingUp.resolve(); + return setUp.promise; + }); + const waitingForPopup = yield* nextPopup(opener.output).pipe(Effect.forkScoped); + contexts[0]!.page.emit("popup", popup.page); + const shown = yield* Fiber.join(waitingForPopup); + yield* Effect.promise(() => settingUp.promise); + + yield* opener.input({ type: "releaseControl" }); + setUp.resolve(); + const popupViewer = yield* browser.attachViewer( + viewerInput(shown.tabId, true, sessionId, shown.handoffId), + ); + let control = yield* Queue.take(popupViewer.output); + while (control._tag !== "control") control = yield* Queue.take(popupViewer.output); + expect(control).toMatchObject({ controller: "agent" }); + + yield* opener.input({ type: "takeControl" }); + const manager = yield* Manager.PreviewManager; + const events = yield* manager.subscribeEvents; + const registering = Promise.withResolvers(); + const register = Promise.withResolvers(); + const registered = Promise.withResolvers(); + const open = manager.open; + using _open = vi.spyOn(manager, "open").mockImplementationOnce((input) => + Effect.promise(() => { + registering.resolve(); + return register.promise; + }).pipe( + Effect.andThen(open(input)), + Effect.tap(() => Effect.sync(() => registered.resolve())), + ), + ); + contexts[0]!.page.emit("popup", makeContext().page); + yield* Effect.promise(() => registering.promise); + yield* opener.input({ type: "releaseControl" }); + register.resolve(); + yield* Effect.promise(() => registered.promise); + let opened = yield* PubSub.take(events); + while (opened.type !== "opened") opened = yield* PubSub.take(events); + const lateViewer = yield* browser.attachViewer( + viewerInput(opened.snapshot.tabId, true, sessionId), + ); + control = yield* Queue.take(lateViewer.output); + while (control._tag !== "control") control = yield* Queue.take(lateViewer.output); + expect(control).toMatchObject({ controller: "agent" }); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("a controlled agent popup opens when its agent is at the tab limit", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, broker, tabId } = yield* ready; + const viewer = yield* browser.attachViewer(viewerInput(tabId, true, "opener-session")); + yield* viewer.input({ type: "takeControl" }); + for (let index = 1; index < 8; index++) { + yield* broker.invoke({ + scope, + tabId, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }); + } + const events = yield* (yield* Manager.PreviewManager).subscribeEvents; + const popup = makeContext(); + contexts[0]!.page.emit("popup", popup.page); + expect(popup.page.close).not.toHaveBeenCalled(); + let opened = yield* PubSub.take(events); + while (opened.type !== "opened" || opened.snapshot.tabId === tabId) { + opened = yield* PubSub.take(events); + } + expect(opened.snapshot.automationOwner).toBeDefined(); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("an uncontrolled agent tab at the limit closes its popup", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + for (let index = 1; index < 8; index++) { + yield* broker.invoke({ + scope, + tabId, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }); + } + const before = (yield* manager.list({ threadId: scope.thread.threadId })).sessions; + expect(before).toHaveLength(8); + + const popup = makeContext(); + contexts[0]!.page.emit("popup", popup.page); + expect(popup.page.close).toHaveBeenCalled(); + expect( + (yield* manager.list({ threadId: scope.thread.threadId })).sessions.map( + (session) => session.tabId, + ), + ).toEqual(before.map((session) => session.tabId)); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("a watcher alone does not reveal or receive an agent popup", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const watcher = yield* browser.attachViewer(viewerInput(tabId, false)); + const events = yield* manager.subscribeEvents; + contexts[0]!.page.emit("popup", makeContext().page); + let opened = yield* PubSub.take(events); + while (opened.type !== "opened") opened = yield* PubSub.take(events); + expect(opened.snapshot).toMatchObject({ reveal: false }); + expect((yield* Queue.clear(watcher.output)).some((item) => item._tag === "popup")).toBe( + false, + ); + }), + ).pipe(Effect.provide(layer)), +); + +it.live( + "releasing control before a popup keeps it hidden and does not switch the released viewer", + () => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const viewer = yield* browser.attachViewer(viewerInput(tabId, true)); + yield* viewer.input({ type: "takeControl" }); + yield* viewer.input({ type: "releaseControl" }); + yield* Queue.clear(viewer.output); + const events = yield* manager.subscribeEvents; + contexts[0]!.page.emit("popup", makeContext().page); + let opened = yield* PubSub.take(events); + while (opened.type !== "opened") opened = yield* PubSub.take(events); + expect(opened.snapshot).toMatchObject({ reveal: false }); + expect((yield* Queue.clear(viewer.output)).some((item) => item._tag === "popup")).toBe( + false, + ); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("a popup stays hidden after its controlling viewer detaches", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const watcher = yield* browser.attachViewer(viewerInput(tabId, false)); + const controllerScope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(controllerScope, Exit.void)); + const controller = yield* browser + .attachViewer(viewerInput(tabId, true, "opener-session")) + .pipe(Effect.provideService(Scope.Scope, controllerScope)); + yield* controller.input({ type: "takeControl" }); + yield* Queue.clear(controller.output); + yield* Scope.close(controllerScope, Exit.void); + + const popup = makeContext(); + const events = yield* manager.subscribeEvents; + contexts[0]!.page.emit("popup", popup.page); + + let opened = yield* PubSub.take(events); + while (opened.type !== "opened") opened = yield* PubSub.take(events); + expect(opened.snapshot).toMatchObject({ reveal: false }); + expect((yield* Queue.clear(watcher.output)).some((item) => item._tag === "popup")).toBe( + false, + ); + expect((yield* Queue.clear(controller.output)).some((item) => item._tag === "popup")).toBe( + false, + ); + + const popupViewer = yield* browser.attachViewer( + viewerInput(opened.snapshot.tabId, true, "opener-session"), + ); + let control = yield* Queue.take(popupViewer.output); + while (control._tag !== "control") control = yield* Queue.take(popupViewer.output); + expect(control).toMatchObject({ controller: "agent" }); + }), + ).pipe(Effect.provide(layer)), +); + it.live("only the controlling viewer's copy and cut keys reach its clipboard", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 6f3e86a7501ea..e4ceea01d2be5 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -193,7 +193,7 @@ export type ServerBrowserViewerOutput = readonly accept: string; } | { readonly _tag: "fileChooserClosed"; readonly id: string } - | { readonly _tag: "popup"; readonly tabId: string } + | { readonly _tag: "popup"; readonly tabId: string; readonly handoffId?: string } | { readonly _tag: "download"; readonly id: string; @@ -218,6 +218,8 @@ export class ServerBrowser extends Context.Service< readonly maxHeight: number; readonly quality: number; readonly canOperate: boolean; + readonly sessionId?: string; + readonly handoffId?: string | undefined; }) => Effect.Effect< ServerBrowserViewer, ServerBrowserTabNotFoundError | ServerBrowserLaunchError, @@ -246,9 +248,16 @@ export class ServerBrowser extends Context.Service< } >()("t3/preview/ServerBrowser") {} +interface PopupHandoff { + readonly id: string; + readonly sessionId: string | undefined; + active: boolean; +} + interface ViewerState { readonly id: string; readonly canOperate: boolean; + handoff: PopupHandoff; readonly pressedKeys: Map; readonly pressedButtons: Map<"left" | "middle" | "right", { x: number; y: number }>; readonly push: (output: ServerBrowserViewerOutput) => void; @@ -291,6 +300,7 @@ interface ServerTab { agentOperating: boolean; fileChooserIntercepted: boolean; readonly onFileChooser: (chooser: FileChooser) => void; + handoff: PopupHandoff | null; readonly isolatedContext: boolean; @@ -468,7 +478,14 @@ const make = Effect.gen(function* () { const pendingTabs = new Map>(); const closedPendingTabs = new Set(); - const adoptedPages = new Map(); + const adoptedPages = new Map< + string, + { + readonly page: Page; + readonly openerTabId: string; + readonly handoff: PopupHandoff | null; + } + >(); const closedSessions = new Set(); let hostConnectionId: string | null = null; let viewerResizeOrder = 0; @@ -779,6 +796,7 @@ const make = Effect.gen(function* () { agentOperating: false, fileChooserIntercepted: false, onFileChooser: (chooser) => void offerFileChooser(tab, chooser), + handoff: adopted?.handoff ?? null, isolatedContext, desktop: desktop === null ? null : { close: desktop.close }, profileId: snapshot.profileId, @@ -789,7 +807,7 @@ const make = Effect.gen(function* () { setting: snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, colorScheme: "system", zoomFactor: 1, - loading: false, + loading: adopted !== undefined && snapshot.navStatus._tag === "Loading", closing: false, recording: null, recordingStart: null, @@ -921,6 +939,22 @@ const make = Effect.gen(function* () { if (tab.control.agentId !== null) tab.control.track(tab.initialNavigation); else void tab.initialNavigation.catch(constVoid); + } else if (adopted) { + // An adopted page can finish loading before its listeners attach. + let loadReported = false; + page.once("load", () => { + loadReported = true; + }); + const generation = tab.navigationGeneration; + void page + .waitForLoadState("load") + .then(() => { + if (!loadReported && tab.navigationGeneration === generation) { + tab.loading = false; + return reportLoaded(tab); + } + }) + .catch(constVoid); } return tab; }; @@ -1198,34 +1232,42 @@ const make = Effect.gen(function* () { const adoptPopup = async (opener: ServerTab, popup: Page) => { const agentId = opener.control.agentId; - if (opener.closing || (agentId !== null && atTabLimit(agentId))) { + const humanId = opener.control.controller; + const controller = [...opener.viewers].find((viewer) => viewer.id === humanId); + if ( + opener.closing || + (agentId !== null && (controller === undefined ? atTabLimit(agentId) : atServerTabLimit())) + ) { await popup.close().catch(constVoid); return; } const url = popup.url(); + const handoff = controller?.handoff ?? null; - const controller = [...opener.viewers].find( - (viewer) => viewer.id === opener.control.controller, - ); await runPromise( manager.open({ threadId: opener.threadId, ...(/^https?:/i.test(url) ? { url } : {}), runtime: "server", ...(opener.profileId === undefined ? {} : { profileId: opener.profileId }), - ...(opener.control.agentId === null + ...(agentId === null ? {} - : { automationOwner: opener.control.agentId, reveal: false }), + : { + automationOwner: agentId, + // ChatView hides server surfaces with reveal:false, including the one we switch to. + ...(controller === undefined ? { reveal: false } : {}), + }), beforePublish: (snapshot) => adoptedPages.set(tabKey(snapshot.threadId, snapshot.tabId), { page: popup, openerTabId: opener.tabId, + handoff, }), }), ).then( (snapshot) => { - if (opener.control.agentId === null) - controller?.push({ _tag: "popup", tabId: snapshot.tabId }); + if (handoff?.active) + controller?.push({ _tag: "popup", tabId: snapshot.tabId, handoffId: handoff.id }); }, async () => { await popup.close().catch(constVoid); @@ -1271,8 +1313,9 @@ const make = Effect.gen(function* () { }); }); + const atServerTabLimit = () => tabs.size + pendingTabs.size >= SERVER_TAB_LIMIT; const atTabLimit = (agentSessionId: string) => - tabs.size + pendingTabs.size >= SERVER_TAB_LIMIT || + atServerTabLimit() || [...tabs.values()].filter((tab) => tab.control.agentId === agentSessionId).length >= AGENT_TAB_LIMIT; @@ -2307,6 +2350,7 @@ const make = Effect.gen(function* () { const viewer: ViewerState = { id: NodeCrypto.randomUUID(), canOperate: input.canOperate, + handoff: { id: NodeCrypto.randomUUID(), sessionId: input.sessionId, active: true }, pressedKeys: new Map(), pressedButtons: new Map(), push: (next) => { @@ -2348,7 +2392,16 @@ const make = Effect.gen(function* () { reportLiveTabs(); }), ); - if (input.canOperate && tab.control.agentId === null && tab.control.controller === null) { + if ( + input.canOperate && + tab.control.controller === null && + (tab.control.agentId === null || + (tab.handoff?.active && + input.handoffId === tab.handoff.id && + input.sessionId !== undefined && + input.sessionId === tab.handoff.sessionId)) + ) { + tab.handoff = null; yield* Effect.promise(() => tab.control.take(viewer.id)); } broadcastControl(tab); @@ -2425,10 +2478,19 @@ const make = Effect.gen(function* () { try { if (message.type === "takeControl") { const taking = tab.control.take(viewer.id); + if (!viewer.handoff.active) { + viewer.handoff = { + id: NodeCrypto.randomUUID(), + sessionId: input.sessionId, + active: true, + }; + } broadcastControl(tab); await taking; pushFileChooser(tab); } else if (message.type === "releaseControl") { + // A stream disconnect during the popup switch must preserve the grant. + if (viewer.id === tab.control.controller) viewer.handoff.active = false; const releasing = tab.control.release(viewer.id, () => releaseViewerInput(viewer, session), ); diff --git a/apps/server/src/preview/ServerBrowserStream.test.ts b/apps/server/src/preview/ServerBrowserStream.test.ts index d3f4781829a85..9c33dbe388d50 100644 --- a/apps/server/src/preview/ServerBrowserStream.test.ts +++ b/apps/server/src/preview/ServerBrowserStream.test.ts @@ -111,7 +111,7 @@ it.effect.each([ ); const server = Context.get(services, HttpServer.HttpServer); const origin = HttpServer.formatAddress(server.address).replace(/^http/, "ws"); - const resource = `/api/preview-stream/ws?threadId=thread&tabId=tab&wsTicket=one-use-ticket${interactive ? "" : "&interactive=false"}`; + const resource = `/api/preview-stream/ws?threadId=thread&tabId=tab&handoffId=popup-claim&wsTicket=one-use-ticket${interactive ? "" : "&interactive=false"}`; const received = Promise.withResolvers(); const socket = yield* Effect.acquireRelease( Effect.sync(() => new WebSocket(`${origin}${resource}`)), @@ -146,6 +146,8 @@ it.effect.each([ maxHeight: 800, quality: 70, canOperate, + sessionId: "stream-test", + handoffId: "popup-claim", }, ]); expect(auth.requests).toEqual([resource]); diff --git a/apps/server/src/preview/ServerBrowserStream.ts b/apps/server/src/preview/ServerBrowserStream.ts index 8d8164cd911c4..62a5dac819403 100644 --- a/apps/server/src/preview/ServerBrowserStream.ts +++ b/apps/server/src/preview/ServerBrowserStream.ts @@ -85,6 +85,8 @@ const makeHandler = Effect.fnUntraced(function* (browser: ServerBrowser.ServerBr threadId, tabId, canOperate, + sessionId: session.sessionId, + handoffId: params.get("handoffId") ?? undefined, maxWidth: intParam(params, "maxWidth", 1280, 7680), maxHeight: intParam(params, "maxHeight", 800, 4320), quality: intParam(params, "quality", DEFAULT_QUALITY, 100), diff --git a/packages/client-runtime/src/preview/serverBrowserStream.test.ts b/packages/client-runtime/src/preview/serverBrowserStream.test.ts index cfde00363a005..8b63f6c77a6ec 100644 --- a/packages/client-runtime/src/preview/serverBrowserStream.test.ts +++ b/packages/client-runtime/src/preview/serverBrowserStream.test.ts @@ -113,6 +113,29 @@ describe("preview stream downloads", () => { beforeEach(() => vi.stubGlobal("WebSocket", FakeSocket)); afterEach(() => vi.unstubAllGlobals()); + it("carries a targeted popup handoff only to its first interactive stream", () => { + const events = { + onFrame: vi.fn(), + onViewport: vi.fn(), + onConnectedChange: vi.fn(), + onUnauthorized: vi.fn(), + onPopup: vi.fn(), + }; + const opener = createPreviewStreamClient(target, events); + FakeSocket.current.message( + JSON.stringify({ type: "popup", tabId: "popup", handoffId: "claim" }), + ); + expect(events.onPopup).toHaveBeenCalledWith("popup"); + const popup = { ...target, tabId: "popup" }; + const watcher = createPreviewStreamClient({ ...popup, interactive: false }, events); + expect(new URL(FakeSocket.current.url).searchParams.has("handoffId")).toBe(false); + const operator = createPreviewStreamClient(popup, events); + expect(new URL(FakeSocket.current.url).searchParams.get("handoffId")).toBe("claim"); + const later = createPreviewStreamClient(popup, events); + expect(new URL(FakeSocket.current.url).searchParams.has("handoffId")).toBe(false); + for (const client of [opener, watcher, operator, later]) client.stop(); + }); + it("offers a download at a URL carrying the stream's ticket", () => { const onDownload = vi.fn(); createPreviewStreamClient( diff --git a/packages/client-runtime/src/preview/serverBrowserStream.ts b/packages/client-runtime/src/preview/serverBrowserStream.ts index b71fff335f881..f5a06de03ba7e 100644 --- a/packages/client-runtime/src/preview/serverBrowserStream.ts +++ b/packages/client-runtime/src/preview/serverBrowserStream.ts @@ -204,6 +204,10 @@ export interface PreviewStreamClient { readonly stop: () => void; } +const popupHandoffs = new Map(); +const popupKey = (target: PreviewStreamTarget, tabId: string) => + JSON.stringify([target.access.wsBase, target.threadId, tabId]); + const ACK_MESSAGE = JSON.stringify({ type: "ack" }); export function createPreviewStreamClient( target: PreviewStreamTarget, @@ -216,6 +220,12 @@ export function createPreviewStreamClient( maxHeight: String(Math.max(1, Math.round(target.maxHeight))), }); if (target.interactive === false) query.set("interactive", "false"); + else { + const key = popupKey(target, target.tabId); + const handoffId = popupHandoffs.get(key); + popupHandoffs.delete(key); + if (handoffId !== undefined) query.set("handoffId", handoffId); + } const url = withDeviceHubQuery(`${target.access.wsBase}/ws?${query.toString()}`, target.access); let stopped = false; let socket: WebSocket | null = null; @@ -271,6 +281,7 @@ export function createPreviewStreamClient( phase, sequence, tabId, + handoffId, } = message as Record; if ( type === "fileChooser" && @@ -298,6 +309,10 @@ export function createPreviewStreamClient( ) { events.onPointer?.({ phase, x, y, sequence }); } else if (type === "popup" && typeof tabId === "string") { + if (events.onPopup && typeof handoffId === "string") { + popupHandoffs.set(popupKey(target, tabId), handoffId); + if (popupHandoffs.size > 32) popupHandoffs.delete(popupHandoffs.keys().next().value!); + } events.onPopup?.(tabId); } else if (type === "clipboard" && typeof text === "string") { events.onClipboard?.(text);