fix(preview): agents see your browser page, and keep working while you're in another thread

apps/desktop/src/ipc/channels.ts

@@ -111,6 +111,7 @@ export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save";
 export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame";
 export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change";
 export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event";
+export const PREVIEW_FRAME_REQUEST_CHANNEL = "desktop:preview-frame-request";
 export const PREVIEW_OPEN_LINK_CHANNEL = "desktop:preview-open-link";
 
 export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper";

apps/desktop/src/ipc/methods/preview.ts

@@ -54,6 +54,9 @@ export const installPreviewEventForwarding = Effect.fn(
   yield* manager.subscribePointerEvents((event) =>
     electronWindow.sendAll(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, event),
   );
+  yield* manager.subscribeFrameRequests((tabId) =>
+    electronWindow.sendAll(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, tabId),
+  );
   yield* manager.subscribeOpenLinks((event) =>
     electronWindow.sendAll(IpcChannels.PREVIEW_OPEN_LINK_CHANNEL, event),
   );

apps/desktop/src/preload.ts

@@ -429,6 +429,14 @@ contextBridge.exposeInMainWorld("desktopBridge", {
       return () =>
         ipcRenderer.removeListener(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, wrappedListener);
     },
+    onFrameRequest: (listener) => {
+      const wrappedListener = (_event: Electron.IpcRendererEvent, tabId: unknown) => {
+        if (typeof tabId === "string") listener(tabId);
+      };
+      ipcRenderer.on(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener);
+      return () =>
+        ipcRenderer.removeListener(IpcChannels.PREVIEW_FRAME_REQUEST_CHANNEL, wrappedListener);
+    },
     onOpenLink: (listener) => {
       const wrappedListener = (_event: Electron.IpcRendererEvent, linkEvent: unknown) => {
         if (typeof linkEvent !== "object" || linkEvent === null) return;

apps/desktop/src/preview/DesktopBrowserHost.test.ts

@@ -148,4 +148,25 @@ describe("DesktopBrowserHost", () => {
       expect(host.humanStartedDownload(debuggee.tab.webContents)).toBe(false);
     }),
   );
+
+  it.effect("asks the tab to draw when the server waits for a capture", () =>
+    Effect.gen(function* () {
+      const host = yield* DesktopBrowserHost.make;
+      host.attach(key, makeDebuggee().tab);
+      const requests = yield* host.frameRequests.pipe(
+        Stream.take(1),
+        Stream.runCollect,
+        Effect.forkScoped,
+      );
+      yield* Effect.yieldNow;
+      const command = (id: number, method: string) =>
+        host.handleCommandLine(
+          encodeJson({ type: "cdp", ...key, message: encodeJson({ id, method, params: {} }) }),
+        );
+      // Reading the page needs no frame; a capture waits for the next one.
+      yield* command(1, "DOM.getDocument");
+      yield* command(2, "Page.captureScreenshot");
+      expect(yield* Fiber.join(requests)).toEqual([key]);
+    }).pipe(Effect.scoped),
+  );
 });

apps/desktop/src/preview/DesktopBrowserHost.ts

@@ -31,6 +31,8 @@ const lineEncoder = new TextEncoder();
 /** CDP commands through which an agent clicks, types, or navigates the page. */
 const AGENT_INPUT_COMMAND =
   /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/;
+/** A capture waits for the page's next frame, which a hidden tab only draws on request. */
+const CAPTURE_COMMAND = /"method":"Page\.captureScreenshot"/;
 /** How long after an agent's input a download still counts as the agent's. */
 const AGENT_DOWNLOAD_WINDOW_MS = 5_000;
 
@@ -93,6 +95,8 @@ export class DesktopBrowserHost extends Context.Service<
       readonly x: number;
       readonly y: number;
     }>;
+    /** Server tabs whose page the server is waiting to capture, so they must draw. */
+    readonly frameRequests: Stream.Stream<DesktopBrowserTabKey>;
   }
 >()("@t3tools/desktop/preview/DesktopBrowserHost") {}
 
@@ -104,6 +108,7 @@ export const make = Effect.gen(function* () {
     readonly x: number;
     readonly y: number;
   }>(16);
+  const frameRequests = yield* PubSub.sliding<DesktopBrowserTabKey>(16);
   const runFork = Effect.runForkWith(yield* Effect.context<never>());
   const tabs = new Map<string, AttachedTab>();
   const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event));
@@ -204,6 +209,8 @@ export const make = Effect.gen(function* () {
         return;
       }
       if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now();
+      if (CAPTURE_COMMAND.test(command.value.message))
+        runFork(PubSub.publish(frameRequests, tab.key));
       relayFor(tab).receive(command.value.message);
     });
 
@@ -226,6 +233,7 @@ export const make = Effect.gen(function* () {
 
   return DesktopBrowserHost.of({
     pointers: Stream.fromPubSub(pointers),
+    frameRequests: Stream.fromPubSub(frameRequests),
     // Subscribes before announcing, so no attach falls between the two.
     events: Stream.unwrap(
       Effect.gen(function* () {

apps/desktop/src/preview/Manager.ts

@@ -415,6 +415,8 @@ type RecordingInputListener = (event: DesktopPreviewRecordingInputEvent) => Effe
 
 type PointerEventListener = (event: DesktopPreviewPointerEvent) => Effect.Effect<void>;
 
+type FrameRequestListener = (tabId: string) => Effect.Effect<void>;
+
 type OpenLinkListener = (event: DesktopPreviewOpenLinkEvent) => Effect.Effect<void>;
 
 /**
@@ -562,6 +564,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
   const attachedRef = yield* Ref.make<ReadonlyMap<number, ManagedListeners>>(new Map());
   const listenersRef = yield* Ref.make<ReadonlySet<Listener>>(new Set());
   const pointerEventListenersRef = yield* Ref.make<ReadonlySet<PointerEventListener>>(new Set());
+  const frameRequestListenersRef = yield* Ref.make<ReadonlySet<FrameRequestListener>>(new Set());
   const openLinkListenersRef = yield* Ref.make<ReadonlySet<OpenLinkListener>>(new Set());
   const recordingInputListenersRef = yield* Ref.make<ReadonlySet<RecordingInputListener>>(
     new Set(),
@@ -838,6 +841,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
       | "recording-frame"
       | "recording-input"
       | "pointer-event"
+      | "frame-request"
       | "open-link",
     tabId: string,
     delivery: () => Effect.Effect<void>,
@@ -2411,7 +2415,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
         return;
       }
       if (afterAttach.serverTab) {
-        yield* listenForAgentPointers;
+        yield* listenForAgentSignals;
         browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger });
       }
       if (afterAttach.colorScheme !== "system") {
@@ -3343,17 +3347,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
       { discard: true },
     );
   });
-  // Listening starts with the first tab of the desktop's own server.
-  let pointersStarted = false;
-  const listenForAgentPointers = Effect.suspend(() => {
-    if (pointersStarted) return Effect.void;
-    pointersStarted = true;
-    return browserHost.pointers.pipe(
-      Stream.runForEach(emitAgentPointer),
-      Effect.forkIn(parentScope),
-      Effect.asVoid,
+  /** Tells the renderer that the server waits for a frame of this tab, so it must draw. */
+  const emitFrameRequest = Effect.fn("PreviewManager.emitFrameRequest")(function* (key: {
+    readonly threadId: string;
+    readonly tabId: string;
+  }) {
+    const tab = [...(yield* SynchronizedRef.get(tabsRef)).values()].find(
+      (candidate) =>
+        candidate.serverTab?.threadId === key.threadId && candidate.serverTab.tabId === key.tabId,
+    );
+    if (!tab) return;
+    const listeners = yield* Ref.get(frameRequestListenersRef);
+    yield* Effect.forEach(
+      listeners,
+      (listener) => deliverEvent("frame-request", tab.tabId, () => listener(tab.tabId)),
+      { discard: true },
     );
   });
+  // Listening starts with the first tab of the desktop's own server.
+  let agentSignalsStarted = false;
+  const listenForAgentSignals = Effect.suspend(() => {
+    if (agentSignalsStarted) return Effect.void;
+    agentSignalsStarted = true;
+    return Effect.all([
+      browserHost.pointers.pipe(Stream.runForEach(emitAgentPointer), Effect.forkIn(parentScope)),
+      browserHost.frameRequests.pipe(
+        Stream.runForEach(emitFrameRequest),
+        Effect.forkIn(parentScope),
+      ),
+    ]).pipe(Effect.asVoid);
+  });
 
   const emitOpenLink = Effect.fn("PreviewManager.emitOpenLink")(function* (
     event: DesktopPreviewOpenLinkEvent,
@@ -3386,6 +3409,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
       [
         Ref.set(listenersRef, new Set()),
         Ref.set(pointerEventListenersRef, new Set()),
+        Ref.set(frameRequestListenersRef, new Set()),
         Ref.set(openLinkListenersRef, new Set()),
         Ref.set(recordingFrameListenersRef, new Set()),
         Ref.set(recordingInputListenersRef, new Set()),
@@ -3432,6 +3456,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
     stopRecording,
     subscribePointerEvents: (listener: PointerEventListener) =>
       subscribe(pointerEventListenersRef, listener),
+    subscribeFrameRequests: (listener: FrameRequestListener) =>
+      subscribe(frameRequestListenersRef, listener),
     subscribeOpenLinks: (listener: OpenLinkListener) => subscribe(openLinkListenersRef, listener),
     subscribeRecordingInputs: (listener: RecordingInputListener) =>
       subscribe(recordingInputListenersRef, listener),
@@ -3674,6 +3700,9 @@ export class PreviewManager extends Context.Service<
     readonly subscribePointerEvents: (
       listener: PointerEventListener,
     ) => Effect.Effect<void, never, Scope.Scope>;
+    readonly subscribeFrameRequests: (
+      listener: FrameRequestListener,
+    ) => Effect.Effect<void, never, Scope.Scope>;
     readonly subscribeOpenLinks: (
       listener: OpenLinkListener,
     ) => Effect.Effect<void, never, Scope.Scope>;
@@ -3819,6 +3848,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
     saveRecording: operations.saveRecording,
     subscribeStateChanges: operations.subscribeStateChanges,
     subscribePointerEvents: operations.subscribePointerEvents,
+    subscribeFrameRequests: operations.subscribeFrameRequests,
     subscribeOpenLinks: operations.subscribeOpenLinks,
     subscribeRecordingFrames: operations.subscribeRecordingFrames,
     subscribeRecordingInputs: operations.subscribeRecordingInputs,

apps/server/src/preview/ServerBrowser.test.ts

@@ -1568,3 +1568,60 @@ it.live("a desktop page that comes back reconnects without waiting for a viewer"
     }),
   ).pipe(Effect.provide(layer)),
 );
+
+it.live("an agent tab that went headless moves onto the desktop page that mounts late", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      const { browser, broker, tabId } = yield* ready;
+      const headless = contexts[0]!.page;
+      // The agent downloaded a file on the headless page; Chromium writes it, here the test does.
+      const saved = Promise.withResolvers<string>();
+      const written = Promise.withResolvers<void>();
+      headless.emit("download", {
+        failure: async () => null,
+        saveAs: (path: string) => {
+          saved.resolve(path);
+          return written.promise;
+        },
+        suggestedFilename: () => "report.csv",
+        url: () => "blob:http://localhost:5173/1",
+      });
+      const fs = yield* FileSystem.FileSystem;
+      yield* fs.writeFileString(yield* Effect.promise(() => saved.promise), "a,b");
+      written.resolve();
+      const status = () =>
+        broker.invoke<PreviewAutomationStatus>({ scope, tabId, operation: "status", input: {} });
+      let before = yield* status();
+      while (!before.downloads?.length) {
+        yield* Effect.yieldNow;
+        before = yield* status();
+      }
+      // A remote viewer streams the headless page and has stopped reading.
+      const viewer = yield* browser.attachViewer(viewerInput(tabId, false));
+      stallViewer(viewer, { _tag: "viewport", width: 1, height: 1 });
+      while (desktopDetaches.listenerCount("attach") === 0) yield* Effect.yieldNow;
+      // The user opens the thread after the attach wait ran out, so the desktop mounts its page now.
+      desktopTabs.add(tabId);
+      desktopDetaches.emit("attach", { threadId: scope.thread.threadId, tabId });
+      while (desktopConnections.length === 0) yield* Effect.yieldNow;
+      expect(headless.isClosed()).toBe(true);
+      // Even with a full queue, the viewer learns to reconnect to the new page.
+      expect(yield* Queue.take(viewer.output)).toEqual({ _tag: "reconnect" });
+      yield* broker.invoke({
+        scope,
+        tabId,
+        operation: "navigate",
+        input: { url: "http://localhost:5173/next" },
+      });
+      // The agent now drives the page the user sees.
+      const desktopPage = desktopConnections[0]!.context.page;
+      expect(desktopPage.goto.mock.calls.map(([url]) => url)).toContain(
+        "http://localhost:5173/next",
+      );
+      // The session stayed open, so the agent's download is still there.
+      const after = yield* status();
+      expect(after.downloads).toEqual(before.downloads);
+      expect(yield* fs.exists(after.downloads?.[0]?.path ?? "")).toBe(true);
+    }),
+  ).pipe(Effect.provide(layer)),
+);

apps/server/src/preview/ServerBrowser.ts

@@ -683,22 +683,26 @@ const make = Effect.gen(function* () {
   /** Whether the preview session for a tab still exists. */
   const sessionOpen = (tab: ServerTab) => !closedSessions.has(tabKey(tab.threadId, tab.tabId));
 
-  const dropTab = (tab: ServerTab, closeSession: boolean) => {
+  /** `handOff` keeps the session for a new page, so viewers reconnect to it. */
+  const dropTab = (tab: ServerTab, closeSession: boolean, handOff = false) => {
     const key = tabKey(tab.threadId, tab.tabId);
     if (tabs.get(key) !== tab) return;
     tabs.delete(key);
     tab.closing = true;
     clearAbortedNavigation(tab);
     // A desktop page outlives the connection unless its session closed with it.
-    const end = tab.desktop && !closeSession && sessionOpen(tab) ? "reconnect" : "gone";
+    const end =
+      (tab.desktop || handOff) && !closeSession && sessionOpen(tab) ? "reconnect" : "gone";
     for (const viewer of tab.viewers) viewer.push({ _tag: end });
     void tab.control.close().catch(constVoid);
     // The desktop owns its page; letting go only ends this connection.
     if (tab.desktop) void tab.desktop.close().catch(constVoid);
     else void tab.page.close().catch(constVoid);
     if (tab.isolatedContext) void tab.page.context().close().catch(constVoid);
     void tab.recording?.encoder.close().catch(constVoid);
-    void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid);
+    // A handed-off tab keeps its downloads; the folder is keyed by tab, not page.
+    if (!handOff)
+      void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid);
     reportLiveTabs();
     if (closeSession) {
       runFork(manager.close({ threadId: tab.threadId, tabId: tab.tabId }).pipe(Effect.ignore));
@@ -2233,7 +2237,12 @@ const make = Effect.gen(function* () {
           if (next._tag === "frame") runFork(next.ack);
           // State a stalled viewer cannot miss replaces its backlog. It runs
           // synchronously so an older replacement can never land after a newer one.
-          else if (next._tag === "gone" || next._tag === "control" || next._tag === "fileChooser") {
+          else if (
+            next._tag === "gone" ||
+            next._tag === "reconnect" ||
+            next._tag === "control" ||
+            next._tag === "fileChooser"
+          ) {
             const dropped = Effect.runSyncExit(Queue.clear(output));
             if (dropped._tag === "Failure") return;
             Queue.offerUnsafe(output, next);
@@ -2408,15 +2417,28 @@ const make = Effect.gen(function* () {
   // The page came back, for example after its DevTools closed. Reconnect now
   // rather than on the next viewer or agent, or the tab's URL and title stop
   // reaching every client in the meantime.
+  // A tab that went headless before the desktop mounted it (its thread was in
+  // the background) moves onto the desktop's page. Otherwise the user and the
+  // agent see two different pages under one tab.
   yield* desktopChannel.attached.pipe(
     Stream.runForEach((key) =>
       Effect.gen(function* () {
-        if (tabs.has(tabKey(key.threadId, key.tabId))) return;
+        const pending = pendingTabs.get(tabKey(key.threadId, key.tabId));
+        if (pending) yield* Effect.promise(() => pending.catch(constVoid));
+        const existing = tabs.get(tabKey(key.threadId, key.tabId));
+        if (existing?.desktop) return;
+        if (existing) dropTab(existing, false, true);
         const { sessions } = yield* manager.list({ threadId: ThreadId.make(key.threadId) });
         const snapshot = sessions.find(
           (session) => session.tabId === key.tabId && session.runtime === "server",
         );
-        if (snapshot) yield* Effect.promise(() => ensureTab(snapshot).catch(constVoid));
+        const tab = snapshot
+          ? yield* Effect.promise(() => ensureTab(snapshot).catch(() => null))
+          : null;
+        if (!existing) return;
+        if (tab) tab.downloads.unshift(...existing.downloads);
+        else
+          void NodeFSP.rm(downloadDir(existing), { recursive: true, force: true }).catch(constVoid);
       }).pipe(Effect.ignore),
     ),
     Effect.forkScoped,