From 5409cb39f0aadaa31c67c5634759cabdd06d0ac7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 25 Oct 2026 06:42:50 +0000 Subject: [PATCH 01/17] test(desktop): reproduce #15579, relay forces focus emulation onto hidden preview guests Reproduces issue 15579: Playwright enables Emulation.setFocusEmulationEnabled on every page it connects to, and the CDP relay forwards it to the guest. Chromium then treats the guest as captured, so a preview the panel hides never throttles. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/preview/CdpRelay.test.ts | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/apps/desktop/src/preview/CdpRelay.test.ts b/apps/desktop/src/preview/CdpRelay.test.ts index ec314d1f9d478..cf220c0f185b1 100644 --- a/apps/desktop/src/preview/CdpRelay.test.ts +++ b/apps/desktop/src/preview/CdpRelay.test.ts @@ -95,6 +95,31 @@ describe("CDP relay", () => { expect(send.mock.calls).toEqual([["Runtime.runIfWaitingForDebugger", {}, "iframe-session"]]); }); + // Playwright enables focus emulation on every page it connects to, and Chromium then + // counts the guest as captured: a preview the panel hides keeps its timers unthrottled. + it("acknowledges Playwright's focus emulation without forcing it onto the guest", async () => { + const send = vi.fn(async () => ({})); + const written: Array> = []; + const relay = createCdpRelayConnection(makeTarget(send), (raw) => + written.push(JSON.parse(raw)), + ); + relay.receive( + JSON.stringify({ + id: 1, + method: "Emulation.setFocusEmulationEnabled", + params: { enabled: true }, + sessionId: "t3-preview-page", + }), + ); + await settle(); + expect(written).toEqual([{ id: 1, result: {}, sessionId: "t3-preview-page" }]); + expect(send).not.toHaveBeenCalledWith( + "Emulation.setFocusEmulationEnabled", + { enabled: true }, + undefined, + ); + }); + it("routes tab events to the page session once the page is announced", async () => { const written: Array> = []; const relay = createCdpRelayConnection(makeTarget(vi.fn()), (raw) => From dd53cbd28d3b0ea2ea766cca4711174116ca4edf Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 25 Oct 2026 06:50:46 +0000 Subject: [PATCH 02/17] fix(desktop): hidden preview tabs throttle again Playwright enables focus emulation on every page it connects to, and the CDP relay forwarded it to the guest. Chromium counts a focus-emulated page as captured, so a preview the panel hid kept its timers at full rate. The relay now acknowledges Playwright's request without forwarding it. The desktop host emulates focus itself while the server drives the tab (for five seconds after its last command) or streams it with a screencast, and waits for the change before forwarding the command. A released relay ends its screencast hold. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/preview/CdpRelay.ts | 2 + .../src/preview/DesktopBrowserHost.test.ts | 67 +++++++++++++++- .../desktop/src/preview/DesktopBrowserHost.ts | 80 +++++++++++++++++-- 3 files changed, 142 insertions(+), 7 deletions(-) diff --git a/apps/desktop/src/preview/CdpRelay.ts b/apps/desktop/src/preview/CdpRelay.ts index 82264cc4b870b..b45667eb26198 100644 --- a/apps/desktop/src/preview/CdpRelay.ts +++ b/apps/desktop/src/preview/CdpRelay.ts @@ -137,6 +137,8 @@ export function createCdpRelayConnection( if (command.method === "Runtime.runIfWaitingForDebugger" && child === undefined) { return Promise.resolve({}); } + // Playwright asks for this on every page; the host emulates focus only while it is in use. + if (command.method === "Emulation.setFocusEmulationEnabled") return Promise.resolve({}); return target.send(command.method, command.params ?? {}, child); }; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 50039c4e4b04e..8e46f4b7de75f 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -6,6 +6,7 @@ import * as Fiber from "effect/Fiber"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import * as NodeEvents from "node:events"; import * as DesktopBrowserHost from "./DesktopBrowserHost.ts"; @@ -35,13 +36,19 @@ const makeDebuggee = () => { const pending: Array<() => void> = []; const sent: Array = []; const sendWaiters: Array<() => void> = []; + const focusEmulation: Array = []; const debuggee = Object.assign(emitter, { - sendCommand: (method: string, params?: { id?: number }) => + sendCommand: (method: string, params?: { id?: number; enabled?: boolean }) => new Promise((resolve) => { if (method === "Target.getTargetInfo") { resolve({ targetInfo: { targetId: "GUEST" } }); return; } + if (method === "Emulation.setFocusEmulationEnabled") { + focusEmulation.push(params?.enabled === true); + resolve({}); + return; + } sent.push(params?.id); pending.push(() => resolve({ method })); sendWaiters.shift()?.(); @@ -61,12 +68,17 @@ const makeDebuggee = () => { navigate: () => webContents.emit("did-navigate"), /** Ids of the commands the debugger received, in order. */ sent, + focusEmulation, nextSend: () => new Promise((resolve) => sendWaiters.push(resolve)), release: () => pending.splice(0).forEach((resolve) => resolve()), releaseFirst: () => pending.shift()?.(), }; }; +/** Commands wait behind a focus emulation change; it settles within a macrotask. */ +// oxlint-disable-next-line t3code/no-real-time-wait-in-tests -- a zero-delay macrotask boundary, not a wait for time +const settle = Effect.promise(() => new Promise((resolve) => setImmediate(resolve))); + const takeEvents = (host: DesktopBrowserHost.DesktopBrowserHost["Service"], count: number) => host.events.pipe( Stream.take(count), @@ -123,6 +135,7 @@ describe("DesktopBrowserHost", () => { yield* command(1, "Page.captureScreenshot"); yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); yield* command(2, "DOM.enable"); + yield* settle; debuggee.release(); const [, reply] = yield* Fiber.join(reader); @@ -141,6 +154,7 @@ describe("DesktopBrowserHost", () => { yield* command(host, 1, "Page.captureScreenshot"); yield* command(host, 2, "Page.captureScreenshot"); yield* command(host, 3, "DOM.enable"); + yield* settle; expect(debuggee.sent).toEqual([1, 3]); debuggee.releaseFirst(); @@ -159,6 +173,7 @@ describe("DesktopBrowserHost", () => { yield* command(host, 1, "Page.captureScreenshot"); yield* command(host, 2, "Page.captureScreenshot"); yield* command(host, 3, "Page.captureScreenshot"); + yield* settle; debuggee.navigate(); expect(debuggee.sent).toEqual([1, 2]); @@ -181,6 +196,7 @@ describe("DesktopBrowserHost", () => { yield* command(host, 1, "Page.captureScreenshot"); yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); yield* command(host, 3, "Page.captureScreenshot"); expect(debuggee.sent).toEqual([1]); @@ -192,6 +208,55 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect("emulates focus while the server drives a guest, then lets a hidden one throttle", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + + yield* command(host, 1, "DOM.enable"); + yield* settle; + expect(debuggee.focusEmulation).toEqual([true]); + expect(debuggee.sent).toEqual([1]); + + yield* TestClock.adjust("4 seconds"); + yield* command(host, 2, "Input.dispatchMouseEvent"); + yield* TestClock.adjust("4 seconds"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* TestClock.adjust("10 seconds"); + expect(debuggee.focusEmulation).toEqual([true, false]); + + yield* command(host, 3, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.focusEmulation).toEqual([true, false, true]); + expect(debuggee.sent).toEqual([1, 2, 3]); + }), + ); + + it.effect("keeps a streamed guest focused until its screencast ends", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + + yield* command(host, 1, "Page.startScreencast"); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* command(host, 2, "Page.stopScreencast"); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + + // A viewer whose connection went away without stopping its screencast. + yield* command(host, 3, "Page.startScreencast"); + yield* TestClock.adjust("1 minute"); + yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false, true, false]); + }), + ); + it.effect("saves a server tab's download under its CDP guid where the server asked", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 849ae658a8219..adb2ee6e86bfa 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -23,6 +23,7 @@ const lineEncoder = new TextEncoder(); const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; const AGENT_DOWNLOAD_WINDOW_MS = 5_000; +const FOCUS_IDLE = "5 seconds"; export interface DesktopBrowserTabKey { readonly threadId: string; @@ -45,6 +46,11 @@ interface AttachedTab { downloadDirectory: string | null; pendingDownloadGuid: string | null; agentInputAt: number; + focusEmulated: boolean; + focusChange: Promise | null; + focusUses: number; + focusReleasing: boolean; + screencasting: boolean; readonly onMessage: ( event: Electron.Event, method: string, @@ -119,6 +125,56 @@ export const make = Effect.gen(function* () { else tab.waitingScreenshots.push(start); }); + /** + * Chromium counts a focus-emulated guest as captured, so a guest the panel hides would + * never throttle. Without it, a hidden guest paints no frames and focus() fires no focus + * events, so the guest is emulated focused while the server drives it or streams it. + */ + const setFocusEmulation = (tab: AttachedTab, enabled: boolean) => { + if (tab.focusEmulated === enabled) return; + tab.focusEmulated = enabled; + // A guest that refuses it, or a debugger already gone, behaves as unfocused. + const change: Promise = tab.debuggee.debugger + .sendCommand("Emulation.setFocusEmulationEnabled", { enabled }) + .then( + () => undefined, + () => undefined, + ) + .finally(() => { + if (tab.focusChange === change) tab.focusChange = null; + }); + tab.focusChange = change; + }; + + const releaseFocusWhenIdle = (tab: AttachedTab) => { + if (tab.focusReleasing) return; + tab.focusReleasing = true; + runFork( + Effect.gen(function* () { + let uses: number; + do { + uses = tab.focusUses; + yield* Effect.sleep(FOCUS_IDLE); + } while (uses !== tab.focusUses); + tab.focusReleasing = false; + if (!tab.screencasting && tabs.get(keyOf(tab.key)) === tab) setFocusEmulation(tab, false); + }), + ); + }; + + const useFocus = (tab: AttachedTab) => { + tab.focusUses += 1; + setFocusEmulation(tab, true); + releaseFocusWhenIdle(tab); + }; + + /** A released relay's screencast has no viewer left to stop it. */ + const dropRelay = (tab: AttachedTab) => { + tab.relay = null; + tab.screencasting = false; + releaseFocusWhenIdle(tab); + }; + const relayFor = (tab: AttachedTab) => { if (tab.relay) return tab.relay; const { webContents, debugger: debuggee } = tab.debuggee; @@ -128,10 +184,17 @@ export const make = Effect.gen(function* () { : debuggee.sendCommand(method, params, sessionId); const relay: CdpRelayConnection = createCdpRelayConnection( { - send: (method, params, sessionId) => - method === "Page.captureScreenshot" - ? takeScreenshot(tab, relay, () => send(method, params, sessionId)) - : send(method, params, sessionId), + send: (method, params, sessionId) => { + if (method === "Page.startScreencast") tab.screencasting = true; + else if (method === "Page.stopScreencast") tab.screencasting = false; + useFocus(tab); + // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. + const forward = () => + method === "Page.captureScreenshot" + ? takeScreenshot(tab, relay, () => send(method, params, sessionId)) + : send(method, params, sessionId); + return tab.focusChange === null ? forward() : tab.focusChange.then(forward); + }, targetId: () => debuggee .sendCommand("Target.getTargetInfo") @@ -188,6 +251,11 @@ export const make = Effect.gen(function* () { downloadDirectory: null, pendingDownloadGuid: null, agentInputAt: Number.NEGATIVE_INFINITY, + focusEmulated: false, + focusChange: null, + focusUses: 0, + focusReleasing: false, + screencasting: false, onMessage: (_event, method, params, sessionId) => { if (method === "Browser.downloadWillBegin") { const guid = (params as { guid?: unknown } | undefined)?.guid; @@ -219,7 +287,7 @@ export const make = Effect.gen(function* () { return; } if (command.value.type === "release") { - tab.relay = null; + dropRelay(tab); return; } if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now(); @@ -235,7 +303,7 @@ export const make = Effect.gen(function* () { Effect.forEach( [...tabs.values()], (tab) => { - tab.relay = null; + dropRelay(tab); return PubSub.publish(outbox, { type: "attached", ...tab.key }); }, { discard: true }, From 0d96515523205bcb977e6472720e2ac38289ea0f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 25 Oct 2026 06:54:56 +0000 Subject: [PATCH 03/17] test(desktop): reproduce #15579, focus emulation expires under a pending command Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 8e46f4b7de75f..25e6844ec0d7a 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -234,6 +234,27 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect("keeps a guest focused while a command it is running is still pending", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + + // An evaluation awaiting the page's next frame, which a throttled guest never paints. + yield* command(host, 1, "Runtime.evaluate"); + yield* settle; + expect(debuggee.sent).toEqual([1]); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(1); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + it.effect("keeps a streamed guest focused until its screencast ends", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; From 0881cac1d11b53005ab6718a93a85370758d4bd7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 25 Oct 2026 06:58:45 +0000 Subject: [PATCH 04/17] fix(desktop): a preview guest stays focused until its pending commands settle Focus emulation released 5s after a command arrived, even while that command (an evaluation awaiting the next frame) was still pending. Each command now holds focus until it settles, and the idle window starts from completion. A released relay or a new document ends every hold, the same points where the screenshot guard gives up on an unanswered capture, so a command Chromium never answers cannot keep a hidden guest unthrottled. Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 33 ++++++++++++++ .../desktop/src/preview/DesktopBrowserHost.ts | 45 ++++++++++++++----- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 25e6844ec0d7a..38591951df2ad 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -218,9 +218,14 @@ describe("DesktopBrowserHost", () => { yield* settle; expect(debuggee.focusEmulation).toEqual([true]); expect(debuggee.sent).toEqual([1]); + debuggee.release(); + yield* settle; yield* TestClock.adjust("4 seconds"); yield* command(host, 2, "Input.dispatchMouseEvent"); + yield* settle; + debuggee.release(); + yield* settle; yield* TestClock.adjust("4 seconds"); expect(debuggee.focusEmulation).toEqual([true]); @@ -255,6 +260,28 @@ describe("DesktopBrowserHost", () => { }).pipe(Effect.scoped), ); + it.effect("lets a guest throttle once Chromium can no longer answer its pending command", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + + yield* command(host, 1, "Runtime.evaluate"); + yield* settle; + yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + + // A command the old document never answers. + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.sent).toEqual([1, 2]); + debuggee.navigate(); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false, true, false]); + }), + ); + it.effect("keeps a streamed guest focused until its screencast ends", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; @@ -266,11 +293,17 @@ describe("DesktopBrowserHost", () => { expect(debuggee.focusEmulation).toEqual([true]); yield* command(host, 2, "Page.stopScreencast"); + yield* settle; + debuggee.release(); + yield* settle; yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); // A viewer whose connection went away without stopping its screencast. yield* command(host, 3, "Page.startScreencast"); + yield* settle; + debuggee.release(); + yield* settle; yield* TestClock.adjust("1 minute"); yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); yield* TestClock.adjust("1 minute"); diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index adb2ee6e86bfa..f04f883ecafba 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -49,6 +49,7 @@ interface AttachedTab { focusEmulated: boolean; focusChange: Promise | null; focusUses: number; + readonly focusPending: Set; focusReleasing: boolean; screencasting: boolean; readonly onMessage: ( @@ -157,22 +158,41 @@ export const make = Effect.gen(function* () { yield* Effect.sleep(FOCUS_IDLE); } while (uses !== tab.focusUses); tab.focusReleasing = false; - if (!tab.screencasting && tabs.get(keyOf(tab.key)) === tab) setFocusEmulation(tab, false); + // The last pending command to settle starts another idle wait. + if (tab.focusPending.size > 0 || tab.screencasting) return; + if (tabs.get(keyOf(tab.key)) === tab) setFocusEmulation(tab, false); }), ); }; - const useFocus = (tab: AttachedTab) => { + /** A command may wait on the page's next frame, so the guest stays focused until it settles. */ + const holdFocus = (tab: AttachedTab, work: () => Promise) => { + const pending = {}; + tab.focusPending.add(pending); tab.focusUses += 1; setFocusEmulation(tab, true); releaseFocusWhenIdle(tab); + return work().finally(() => { + if (!tab.focusPending.delete(pending)) return; + tab.focusUses += 1; + releaseFocusWhenIdle(tab); + }); + }; + + /** + * Chromium can leave a command unanswered, so the screenshot guard's release points + * also end every hold: a released relay has no one waiting, and a new document answers + * nothing pending from the old one. A released relay's screencast has no viewer to stop it. + */ + const dropFocusHolds = (tab: AttachedTab) => { + tab.focusPending.clear(); + releaseFocusWhenIdle(tab); }; - /** A released relay's screencast has no viewer left to stop it. */ const dropRelay = (tab: AttachedTab) => { tab.relay = null; tab.screencasting = false; - releaseFocusWhenIdle(tab); + dropFocusHolds(tab); }; const relayFor = (tab: AttachedTab) => { @@ -187,13 +207,16 @@ export const make = Effect.gen(function* () { send: (method, params, sessionId) => { if (method === "Page.startScreencast") tab.screencasting = true; else if (method === "Page.stopScreencast") tab.screencasting = false; - useFocus(tab); // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. - const forward = () => - method === "Page.captureScreenshot" - ? takeScreenshot(tab, relay, () => send(method, params, sessionId)) - : send(method, params, sessionId); - return tab.focusChange === null ? forward() : tab.focusChange.then(forward); + const forward = () => { + const change = tab.focusChange; + const run = () => send(method, params, sessionId); + return change === null ? run() : change.then(run); + }; + // A screenshot holds focus only once its turn starts, not while queued behind another. + return method === "Page.captureScreenshot" + ? takeScreenshot(tab, relay, () => holdFocus(tab, forward)) + : holdFocus(tab, forward); }, targetId: () => debuggee @@ -254,6 +277,7 @@ export const make = Effect.gen(function* () { focusEmulated: false, focusChange: null, focusUses: 0, + focusPending: new Set(), focusReleasing: false, screencasting: false, onMessage: (_event, method, params, sessionId) => { @@ -267,6 +291,7 @@ export const make = Effect.gen(function* () { // stay unanswered. A crashed page lands here once it reloads, well after its renderer died. onNewDocument: () => { if (tab.screenshotTurn !== null) nextScreenshot(tab); + dropFocusHolds(tab); }, }; tabs.set(id, tab); From 3d9f3a47da55a616a1ee82fe0bb37e6f2d276ba5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 25 Oct 2026 07:09:05 +0000 Subject: [PATCH 05/17] fix(desktop): an agent's preview tab stays focused for its whole turn The server tells the desktop when a thread's agent turn starts and ends, model thinking included and background commands excluded. Tabs the agent uses during the turn stay focused through its pauses between tools. The turn's end also ends holds on commands Chromium never answered, such as a timed-out screenshot of a hidden guest, and a detached CDP session (a closed viewer) drops the holds of commands it sent. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/preview/CdpRelay.test.ts | 16 ++- apps/desktop/src/preview/CdpRelay.ts | 9 +- .../src/preview/DesktopBrowserHost.test.ts | 76 ++++++++++++ .../desktop/src/preview/DesktopBrowserHost.ts | 62 ++++++++-- .../RunExecutionService.test.ts | 111 +++++++++++++++++- .../orchestration-v2/RunExecutionService.ts | 23 +++- .../src/orchestration-v2/runtimeLayer.ts | 2 + .../testkit/ProviderReplayHarness.ts | 2 + .../src/preview/DesktopBrowserChannel.ts | 4 + .../src/preview/PreviewAgentActivity.ts | 58 +++++++++ apps/server/src/preview/ServerBrowser.test.ts | 26 ++++ apps/server/src/preview/ServerBrowser.ts | 7 ++ apps/server/src/server.ts | 8 +- packages/contracts/src/desktopBrowser.ts | 6 + 14 files changed, 387 insertions(+), 23 deletions(-) create mode 100644 apps/server/src/preview/PreviewAgentActivity.ts diff --git a/apps/desktop/src/preview/CdpRelay.test.ts b/apps/desktop/src/preview/CdpRelay.test.ts index cf220c0f185b1..de03ae90dc0ca 100644 --- a/apps/desktop/src/preview/CdpRelay.test.ts +++ b/apps/desktop/src/preview/CdpRelay.test.ts @@ -65,8 +65,14 @@ describe("CDP relay", () => { ]); pending[0]!(); await settle(); - expect(send).toHaveBeenNthCalledWith(1, "Page.captureScreenshot", {}, undefined); - expect(send).toHaveBeenNthCalledWith(2, "DOM.enable", {}, "iframe-session"); + expect(send).toHaveBeenNthCalledWith( + 1, + "Page.captureScreenshot", + {}, + undefined, + "t3-preview-page", + ); + expect(send).toHaveBeenNthCalledWith(2, "DOM.enable", {}, "iframe-session", "iframe-session"); expect(written.at(-1)).toEqual({ id: 5, result: { method: "Page.captureScreenshot" }, @@ -92,7 +98,9 @@ describe("CDP relay", () => { }), ); await settle(); - expect(send.mock.calls).toEqual([["Runtime.runIfWaitingForDebugger", {}, "iframe-session"]]); + expect(send.mock.calls).toEqual([ + ["Runtime.runIfWaitingForDebugger", {}, "iframe-session", "iframe-session"], + ]); }); // Playwright enables focus emulation on every page it connects to, and Chromium then @@ -172,7 +180,7 @@ describe("CDP relay extra sessions", () => { expect(written.find((m) => m["id"] === 4)).toHaveProperty("error"); relay.receive(JSON.stringify({ id: 5, method: "Page.startScreencast", sessionId: extra })); await settle(); - expect(send).toHaveBeenLastCalledWith("Page.startScreencast", {}, undefined); + expect(send).toHaveBeenLastCalledWith("Page.startScreencast", {}, undefined, extra); written.length = 0; relay.event("Page.screencastFrame", { data: "x" }, undefined); expect(written.map((m) => m["sessionId"])).toEqual(["t3-preview-page", extra]); diff --git a/apps/desktop/src/preview/CdpRelay.ts b/apps/desktop/src/preview/CdpRelay.ts index b45667eb26198..206c1d6c389a0 100644 --- a/apps/desktop/src/preview/CdpRelay.ts +++ b/apps/desktop/src/preview/CdpRelay.ts @@ -5,7 +5,10 @@ export interface CdpRelayTarget { method: string, params: Record, sessionId: string | undefined, + /** The relay session that sent it; a detached one waits on nothing it sent. */ + owner?: string, ) => Promise; + readonly sessionDetached?: (owner: string) => void; /** The page's own CDP target id; Playwright keys the main frame by it. */ readonly targetId: () => Promise; readonly url: () => string; @@ -111,8 +114,10 @@ export function createCdpRelayConnection( } case "Target.detachFromTarget": { const sessionId = command.params?.["sessionId"]; - if (typeof sessionId === "string" && sessionId !== PAGE_SESSION_ID) + if (typeof sessionId === "string" && sessionId !== PAGE_SESSION_ID) { sessions.delete(sessionId); + target.sessionDetached?.(sessionId); + } return {}; } case "Target.setAutoAttach": { @@ -139,7 +144,7 @@ export function createCdpRelayConnection( } // Playwright asks for this on every page; the host emulates focus only while it is in use. if (command.method === "Emulation.setFocusEmulationEnabled") return Promise.resolve({}); - return target.send(command.method, command.params ?? {}, child); + return target.send(command.method, command.params ?? {}, child, command.sessionId); }; return { diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 38591951df2ad..3a82b8afabf27 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -282,6 +282,82 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect("keeps a guest the agent used focused through its turn's pauses", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + yield* turn(true); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([]); + + yield* command(host, 1, "Input.dispatchKeyEvent"); + yield* settle; + debuggee.release(); + yield* settle; + // The model thinks between steps; a focus-sensitive menu must not blur meanwhile. + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* turn(false); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }), + ); + + it.effect( + "ends a command's hold with the turn that sent it, even if Chromium never answers", + () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + yield* host.handleCommandLine( + encodeJson({ type: "turn", threadId: key.threadId, active: true }), + ); + + // A screenshot of a hidden guest the server already gave up on. + yield* command(host, 1, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.sent).toEqual([1]); + yield* host.handleCommandLine( + encodeJson({ type: "turn", threadId: key.threadId, active: false }), + ); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }), + ); + + it.effect("lets a guest throttle once the session waiting on its command detaches", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + // A viewer's own CDP session, closed while its still of a hidden guest is pending. + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" }); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* raw({ + id: 3, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + it.effect("keeps a streamed guest focused until its screencast ends", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index f04f883ecafba..2584bff83ecd5 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -49,7 +49,10 @@ interface AttachedTab { focusEmulated: boolean; focusChange: Promise | null; focusUses: number; - readonly focusPending: Set; + /** Each pending command, by the relay session waiting on it. */ + readonly focusPending: Map; + /** Used by the agent during its thread's current turn. */ + turnHeld: boolean; focusReleasing: boolean; screencasting: boolean; readonly onMessage: ( @@ -89,6 +92,8 @@ export const make = Effect.gen(function* () { }>(16); const runFork = Effect.runForkWith(yield* Effect.context()); const tabs = new Map(); + /** Threads whose agent turn runs, by the server's account. */ + const agentTurns = new Set(); const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event)); const nextScreenshot = (tab: AttachedTab) => { @@ -159,16 +164,18 @@ export const make = Effect.gen(function* () { } while (uses !== tab.focusUses); tab.focusReleasing = false; // The last pending command to settle starts another idle wait. - if (tab.focusPending.size > 0 || tab.screencasting) return; + if (tab.focusPending.size > 0 || tab.screencasting || tab.turnHeld) return; if (tabs.get(keyOf(tab.key)) === tab) setFocusEmulation(tab, false); }), ); }; /** A command may wait on the page's next frame, so the guest stays focused until it settles. */ - const holdFocus = (tab: AttachedTab, work: () => Promise) => { + const holdFocus = (tab: AttachedTab, owner: string | undefined, work: () => Promise) => { const pending = {}; - tab.focusPending.add(pending); + tab.focusPending.set(pending, owner); + // The agent pauses to think between steps; its tab stays focused until the turn ends. + if (agentTurns.has(tab.key.threadId)) tab.turnHeld = true; tab.focusUses += 1; setFocusEmulation(tab, true); releaseFocusWhenIdle(tab); @@ -192,9 +199,31 @@ export const make = Effect.gen(function* () { const dropRelay = (tab: AttachedTab) => { tab.relay = null; tab.screencasting = false; + tab.turnHeld = false; dropFocusHolds(tab); }; + /** A detached session, such as a closed viewer, waits on nothing it sent. */ + const dropSessionHolds = (tab: AttachedTab, owner: string) => { + for (const [pending, sender] of tab.focusPending) { + if (sender === owner) tab.focusPending.delete(pending); + } + releaseFocusWhenIdle(tab); + }; + + /** + * A command the server stopped waiting on, such as a timed-out screenshot of a hidden + * guest, may never settle; the end of the turn that sent it ends its hold too. + */ + const endAgentTurn = (threadId: string) => { + agentTurns.delete(threadId); + for (const tab of tabs.values()) { + if (tab.key.threadId !== threadId) continue; + tab.turnHeld = false; + dropFocusHolds(tab); + } + }; + const relayFor = (tab: AttachedTab) => { if (tab.relay) return tab.relay; const { webContents, debugger: debuggee } = tab.debuggee; @@ -204,7 +233,7 @@ export const make = Effect.gen(function* () { : debuggee.sendCommand(method, params, sessionId); const relay: CdpRelayConnection = createCdpRelayConnection( { - send: (method, params, sessionId) => { + send: (method, params, sessionId, owner) => { if (method === "Page.startScreencast") tab.screencasting = true; else if (method === "Page.stopScreencast") tab.screencasting = false; // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. @@ -215,9 +244,10 @@ export const make = Effect.gen(function* () { }; // A screenshot holds focus only once its turn starts, not while queued behind another. return method === "Page.captureScreenshot" - ? takeScreenshot(tab, relay, () => holdFocus(tab, forward)) - : holdFocus(tab, forward); + ? takeScreenshot(tab, relay, () => holdFocus(tab, owner, forward)) + : holdFocus(tab, owner, forward); }, + sessionDetached: (owner) => dropSessionHolds(tab, owner), targetId: () => debuggee .sendCommand("Target.getTargetInfo") @@ -277,7 +307,8 @@ export const make = Effect.gen(function* () { focusEmulated: false, focusChange: null, focusUses: 0, - focusPending: new Set(), + focusPending: new Map(), + turnHeld: false, focusReleasing: false, screencasting: false, onMessage: (_event, method, params, sessionId) => { @@ -304,6 +335,11 @@ export const make = Effect.gen(function* () { Effect.sync(() => { const command = decodeCommand(line); if (Option.isNone(command)) return; + if (command.value.type === "turn") { + if (command.value.active) agentTurns.add(command.value.threadId); + else endAgentTurn(command.value.threadId); + return; + } const tab = tabs.get(keyOf(command.value)); if (!tab) return; if (command.value.type === "pointer") { @@ -324,16 +360,18 @@ export const make = Effect.gen(function* () { return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_DOWNLOAD_WINDOW_MS; }; - const announceAll = Effect.suspend(() => - Effect.forEach( + const announceAll = Effect.suspend(() => { + // A new backend: the old one's turns ended without telling us. + agentTurns.clear(); + return Effect.forEach( [...tabs.values()], (tab) => { dropRelay(tab); return PubSub.publish(outbox, { type: "attached", ...tab.key }); }, { discard: true }, - ), - ); + ); + }); return DesktopBrowserHost.of({ pointers: Stream.fromPubSub(pointers), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 1728e3a83ab03..86f68e593dc37 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -46,6 +46,7 @@ import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter" import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; +import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts"; import * as RunExecutionService from "./RunExecutionService.ts"; import * as RunFinalizationService from "./RunFinalizationService.ts"; import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; @@ -65,6 +66,7 @@ const layerRunExecutionTest = RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({}), @@ -878,6 +880,7 @@ it.effect("passes the thread's MCP app context to the provider under a safe key" RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( + PreviewAgentActivity.layer, Layer.mock(McpAppModelContext.McpAppModelContext)({ forThread: () => Effect.succeed([ @@ -919,6 +922,7 @@ it.effect("fails the run when its ownership check cannot be read before calling Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({ @@ -1120,6 +1124,7 @@ it.effect("starts the provider when checkpoint baseline capture fails", () => Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.fail( @@ -1239,6 +1244,7 @@ it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as co Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => scenario === "start-guard" ? Effect.void : Effect.die("not reached"), @@ -1410,6 +1416,7 @@ it.effect("ingests new member threads of an owned child after the root turn term Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({ @@ -1658,6 +1665,7 @@ it.effect("ends a nested native subagent and its thread when the root run is int Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({ @@ -1960,6 +1968,7 @@ it.effect( Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void, }), @@ -2769,6 +2778,95 @@ it.effect("does not interrupt a provider when ingestion fails before the root tu }), ); +// The server browser keeps an agent's tab focused for exactly this span. +it.effect.each(["completed", "failed"] as const)( + "reports the thread executing until its %s run's event consumer ends", + (outcome) => + Effect.gen(function* () { + const gate = yield* Deferred.make(); + const { previewActivityEnded } = yield* captureRootRunTermination({ + key: `preview-activity:${outcome}`, + shouldFinalizeRun: () => Effect.succeed(true), + events: (ids) => + Stream.fromEffect(Deferred.await(gate)).pipe( + Stream.flatMap(() => + outcome === "completed" + ? Stream.make(rootTerminalEvent(ids, "completed")) + : Stream.fail( + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver, + providerSessionId: ProviderSessionId.make("session:exited"), + cause: "provider process exited", + }), + ), + ), + ), + afterStart: () => + Effect.gen(function* () { + const activity = yield* PreviewAgentActivity.PreviewAgentActivity; + assert.isTrue( + yield* activity.active(ThreadId.make(`thread:preview-activity:${outcome}`)), + ); + yield* Deferred.succeed(gate, undefined); + }), + }); + assert.isTrue(previewActivityEnded); + }), +); + +// Background commands keep the consumer alive for the whole session; only the agent's own +// turn and the children it drives should hold the browser tab focused. +it.effect.each([ + { name: "ends once the root turn finishes beside background work", openSubagent: false }, + { name: "stays while a run-owned subagent continues", openSubagent: true }, +])("preview activity $name", ({ openSubagent }) => + Effect.gen(function* () { + const key = `preview-activity:background:${openSubagent}`; + const consumerWaiting = yield* Deferred.make(); + const release = yield* Deferred.make(); + const providerInstanceId = ProviderInstanceId.make("codex"); + yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + hasPendingBackgroundWorkForThread: () => Effect.succeed(true), + events: (ids) => + Stream.concat( + Stream.fromIterable([ + ...(openSubagent + ? [ + { + type: "subagent.updated", + driver, + subagent: makeRunOwnedSubagentFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + }), + } as const, + ] + : []), + rootTerminalEvent(ids, "completed"), + ] satisfies ReadonlyArray), + Stream.fromEffect( + Deferred.succeed(consumerWaiting, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + ).pipe(Stream.drain), + ), + afterStart: (closed) => + Effect.gen(function* () { + yield* Deferred.await(consumerWaiting); + assert.isFalse(yield* Deferred.isDone(closed)); + const activity = yield* PreviewAgentActivity.PreviewAgentActivity; + assert.equal(yield* activity.active(ThreadId.make(`thread:${key}`)), openSubagent); + yield* Deferred.succeed(release, undefined); + }), + }); + }), +); + it.effect("fails the run when its event stream ends without a root terminal", () => Effect.gen(function* () { const { observed, finalEvents } = yield* captureRootRunTermination({ @@ -2838,6 +2936,7 @@ function captureRootRunTermination(input: { readonly shouldFinalizeRun: () => Effect.Effect; readonly rejectTerminalWrite?: boolean; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; + readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"]; readonly seedOpenSubagent?: boolean; readonly events?: ( ids: BackgroundScenarioIds, @@ -2859,7 +2958,9 @@ function captureRootRunTermination(input: { readonly beforeTerminalWrite?: ( request: Parameters[0], ) => Effect.Effect; - readonly afterStart?: (closed: Deferred.Deferred) => Effect.Effect; + readonly afterStart?: ( + closed: Deferred.Deferred, + ) => Effect.Effect; }) { return Effect.gen(function* () { const ids = backgroundScenarioIds(input.key); @@ -2946,9 +3047,10 @@ function captureRootRunTermination(input: { }), ), ), + Layer.provideMerge(PreviewAgentActivity.layer), ); - yield* Effect.gen(function* () { + const previewActivityEnded = yield* Effect.gen(function* () { const runExecution = yield* RunExecutionService.RunExecutionServiceV2; const start = runExecution.startRootRun({ commandId: CommandId.make(`command:${input.key}`), @@ -2989,6 +3091,7 @@ function captureRootRunTermination(input: { }), startTurn: input.startTurn ?? (() => Effect.void), interruptTurn: input.interruptTurn ?? (() => Effect.void), + hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread, } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime, run: { id: ids.runId, @@ -3050,9 +3153,12 @@ function captureRootRunTermination(input: { ); yield* input.afterStart?.(ingestionDone) ?? Effect.void; yield* Deferred.await(ingestionDone); + const activity = yield* PreviewAgentActivity.PreviewAgentActivity; + return !(yield* activity.active(ids.threadId)); }).pipe(Effect.provide(layerTest)); return { + previewActivityEnded, written: yield* Ref.get(writtenItems), finalEvents: yield* Ref.get(finalEvents), observed: yield* Ref.get(observed), @@ -3401,6 +3507,7 @@ function runBackgroundItemScenario( Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({ diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index d7789df380b68..3d2c512a7970a 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -39,6 +39,7 @@ import * as Stream from "effect/Stream"; import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; +import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as CheckpointService from "./CheckpointService.ts"; import * as EventSink from "./EventSink.ts"; @@ -595,6 +596,7 @@ export const layer: Layer.Layer< | ProviderEventIngestor.ProviderEventIngestorV2 | ServerSettings.ServerSettingsService | McpAppModelContext.McpAppModelContext + | PreviewAgentActivity.PreviewAgentActivity | TurnItemPositionStore.TurnItemPositionStoreV2 > = Layer.effect( RunExecutionServiceV2, @@ -606,6 +608,7 @@ export const layer: Layer.Layer< const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; const serverSettings = yield* ServerSettings.ServerSettingsService; const mcpAppModelContext = yield* McpAppModelContext.McpAppModelContext; + const previewAgentActivity = yield* PreviewAgentActivity.PreviewAgentActivity; const turnItemPositions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; const finalizationObserver = yield* RunFinalizationService.RunFinalizationObserver; @@ -1220,7 +1223,7 @@ export const layer: Layer.Layer< } } }); - const shouldStopProviderEventIngestion = Effect.gen(function* () { + const agentTurnsSettled = Effect.gen(function* () { if (!(yield* Ref.get(rootTerminalSeen))) { return false; } @@ -1237,9 +1240,20 @@ export const layer: Layer.Layer< return false; } const childSubagents = yield* Ref.get(activeChildSubagents); - if (childSubagents.size > 0) { + return childSubagents.size === 0; + }); + const shouldStopProviderEventIngestion = Effect.gen(function* () { + if (!(yield* agentTurnsSettled)) { return false; } + const terminal = yield* Ref.get(terminalEvent); + if ( + terminal !== null && + terminal.status !== "completed" && + !childTurnsContinue(terminal) + ) { + return true; + } // Newly owned items depend on adapters emitting a non-terminal event before the root // terminal. Owner loss (a newer run claimed lastRunOrdinal) must not close this stream // while these sets are non-empty: turn_item.updated writes are not ownership-gated, so @@ -1269,6 +1283,9 @@ export const layer: Layer.Layer< // Held text writes nothing, so its item would take a position only // when finally written, behind every item that began after it. const reservedHeldTurnItemIds = new Set(); + // Ends once the root turn and the children it drives settle. Background commands such + // as a dev server keep this consumer alive afterwards and must not pin preview focus. + const endPreviewActivity = yield* previewAgentActivity.begin(input.run.threadId); const providerEventFiber = yield* eventSubscription.events.pipe( Stream.mapEffect((event) => Effect.gen(function* () { @@ -1442,6 +1459,7 @@ export const layer: Layer.Layer< yield* finalizeRootRun(event); } yield* trackChildLifecycle(event, deliveredEvent !== null); + if (yield* agentTurnsSettled) yield* endPreviewActivity; }), ), Stream.takeUntilEffect(() => shouldStopProviderEventIngestion), @@ -1574,6 +1592,7 @@ export const layer: Layer.Layer< ); }), ), + Effect.ensuring(endPreviewActivity), Effect.ensuring(eventSubscription.close), Effect.forkDetach, ); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index 8e1c65443ea3c..eb6f9d1d3eaa1 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -6,6 +6,7 @@ import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as OrchestrationCommandReceipts from "../persistence/OrchestrationCommandReceipts.ts"; import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts"; import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; +import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts"; import * as McpAppRequests from "../mcpApps/McpAppRequests.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; import * as TextGeneration from "../textGeneration/TextGeneration.ts"; @@ -153,6 +154,7 @@ const layerRunExecutionServiceProvided = RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( McpAppModelContext.layer, + PreviewAgentActivity.layer, layerCheckpointServiceProvided, layerEventSinkProvided, IdAllocator.layer, diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 76f89af4847d1..be25176dc3f1b 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -48,6 +48,7 @@ import * as ProviderTurnControlService from "../ProviderTurnControlService.ts"; import * as ProviderTurnStartService from "../ProviderTurnStartService.ts"; import * as ProviderTurnStartServiceTestkit from "../ProviderTurnStartService.testkit.ts"; import * as McpAppModelContext from "../../mcpApps/McpAppModelContext.ts"; +import * as PreviewAgentActivity from "../../preview/PreviewAgentActivity.ts"; import * as RunExecutionService from "../RunExecutionService.ts"; import * as RunFinalizationService from "../RunFinalizationService.ts"; import * as ThreadTitleRegenerationService from "../ThreadTitleRegenerationService.ts"; @@ -378,6 +379,7 @@ export function layerWithRegistry( Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, layerCheckpointServiceProvided, layerEventSinkProvided, IdAllocator.layer, diff --git a/apps/server/src/preview/DesktopBrowserChannel.ts b/apps/server/src/preview/DesktopBrowserChannel.ts index 4d8ab5e098c3e..a0ceaf6f369fa 100644 --- a/apps/server/src/preview/DesktopBrowserChannel.ts +++ b/apps/server/src/preview/DesktopBrowserChannel.ts @@ -49,6 +49,8 @@ export class DesktopBrowserChannel extends Context.Service< key: DesktopTabKey, pointer: { readonly phase: "move" | "click"; readonly x: number; readonly y: number }, ) => Effect.Effect; + /** Never fails: an undelivered change leaves the guest to its idle release. */ + readonly turn: (threadId: string, active: boolean) => Effect.Effect; } >()("t3/preview/DesktopBrowserChannel") {} @@ -70,6 +72,7 @@ const make = Effect.gen(function* () { isAttached: () => Effect.succeed(false), endpoint: () => Effect.die("No desktop app is attached to this server."), pointer: () => Effect.void, + turn: () => Effect.void, }); } @@ -207,6 +210,7 @@ const make = Effect.gen(function* () { isAttached: (key) => Effect.sync(() => attachedTabs.has(keyOf(key))), endpoint, pointer: (key, pointer) => command({ type: "pointer", ...key, ...pointer }), + turn: (threadId, active) => command({ type: "turn", threadId, active }), }); }); diff --git a/apps/server/src/preview/PreviewAgentActivity.ts b/apps/server/src/preview/PreviewAgentActivity.ts new file mode 100644 index 0000000000000..2e7d261de9e57 --- /dev/null +++ b/apps/server/src/preview/PreviewAgentActivity.ts @@ -0,0 +1,58 @@ +import type { ThreadId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import type * as Scope from "effect/Scope"; + +export interface PreviewAgentTurnChange { + readonly threadId: ThreadId; + readonly active: boolean; +} + +/** + * Whether an agent turn runs on a thread, model thinking included, so its preview tab + * stays focused between tool calls. Background commands outlive the turn and do not count. + */ +export class PreviewAgentActivity extends Context.Service< + PreviewAgentActivity, + { + /** Marks the thread active until the returned effect runs; it runs at most once. */ + readonly begin: (threadId: ThreadId) => Effect.Effect>; + readonly active: (threadId: ThreadId) => Effect.Effect; + /** Each thread's transitions between no turn and some turn, from subscription on. */ + readonly changes: Effect.Effect< + PubSub.Subscription, + never, + Scope.Scope + >; + } +>()("t3/preview/PreviewAgentActivity") {} + +const make = Effect.gen(function* () { + const changes = yield* PubSub.unbounded(); + // A token per turn, so overlapping turns on one thread end only when the last does. + const turns = new Map>(); + + const begin = Effect.fn("PreviewAgentActivity.begin")(function* (threadId: ThreadId) { + const existing = turns.get(threadId); + const tokens = existing ?? new Set(); + if (existing === undefined) turns.set(threadId, tokens); + const token = Symbol(threadId); + tokens.add(token); + if (existing === undefined) yield* PubSub.publish(changes, { threadId, active: true }); + return Effect.suspend(() => { + if (!tokens.delete(token) || tokens.size > 0) return Effect.void; + if (turns.get(threadId) === tokens) turns.delete(threadId); + return PubSub.publish(changes, { threadId, active: false }).pipe(Effect.asVoid); + }); + }); + + return PreviewAgentActivity.of({ + begin, + active: (threadId) => Effect.sync(() => turns.has(threadId)), + changes: PubSub.subscribe(changes), + }); +}); + +export const layer = Layer.effect(PreviewAgentActivity, make); diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 0a9118e7b6d8b..c8061833a80ec 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -32,6 +32,7 @@ import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as Broker from "../mcp/PreviewAutomationBroker.ts"; import * as DesktopChannel from "./DesktopBrowserChannel.ts"; import * as Manager from "./Manager.ts"; +import * as PreviewAgentActivity from "./PreviewAgentActivity.ts"; import * as ServerBrowser from "./ServerBrowser.ts"; import * as PreviewBrowser from "./PreviewBrowser.ts"; @@ -275,6 +276,7 @@ const desktopRenders = (tabId: string) => { return desktopTabs.has(tabId); }; const releasedDesktopTabs: Array = []; +const desktopTurns = new NodeEvents.EventEmitter(); const desktopConnections: Array<{ endpoint: string; context: ReturnType }> = []; const testThread = { threadId: ThreadId.make("browser-test-thread"), @@ -335,7 +337,9 @@ const dependencies = Layer.mergeAll( Effect.sync(() => releasedDesktopTabs.push(key.tabId)), ), pointer: () => Effect.void, + turn: (threadId, active) => Effect.sync(() => desktopTurns.emit("turn", threadId, active)), }), + PreviewAgentActivity.layer, ).pipe( Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-server-browser-" })), Layer.provideMerge(NodeServices.layer), @@ -378,6 +382,28 @@ beforeEach(() => { desktopConnections.length = 0; }); +it.live("tells the desktop once per thread when its agent turns start and end", () => + Effect.gen(function* () { + yield* ServerBrowser.ServerBrowser; + const activity = yield* PreviewAgentActivity.PreviewAgentActivity; + const threadId = scope.thread.threadId; + const turns = yield* Queue.unbounded(); + const record = (id: string, active: boolean) => Queue.offerUnsafe(turns, [id, active]); + desktopTurns.on("turn", record); + yield* Effect.addFinalizer(() => Effect.sync(() => desktopTurns.off("turn", record))); + const endFirst = yield* activity.begin(threadId); + // An overlapping turn on the same thread neither starts nor ends focus on its own. + const endSecond = yield* activity.begin(threadId); + expect(yield* Queue.take(turns)).toEqual([threadId, true]); + yield* endFirst; + yield* endFirst; + expect(yield* activity.active(threadId)).toBe(true); + yield* endSecond; + expect(yield* Queue.take(turns)).toEqual([threadId, false]); + expect(yield* Queue.size(turns)).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + it.live("preview_open reports its navigation timeout instead of the request budget", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 574f843b1991c..93f6849bbb995 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -77,6 +77,7 @@ import { resolveRootCliCommand } from "../cli/invocation.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as PreviewAutomationBroker from "../mcp/PreviewAutomationBroker.ts"; import * as DesktopBrowserChannel from "./DesktopBrowserChannel.ts"; +import * as PreviewAgentActivity from "./PreviewAgentActivity.ts"; import * as PreviewManager from "./Manager.ts"; import * as ServerBrowserPage from "./ServerBrowserPage.ts"; import * as PreviewBrowser from "./PreviewBrowser.ts"; @@ -454,6 +455,7 @@ const make = Effect.gen(function* () { const environment = yield* ServerEnvironment.ServerEnvironment; const previewBrowser = yield* PreviewBrowser.PreviewBrowser; const desktopChannel = yield* DesktopBrowserChannel.DesktopBrowserChannel; + const agentTurns = yield* (yield* PreviewAgentActivity.PreviewAgentActivity).changes; const runFork = Effect.runForkWith(yield* Effect.context()); const launchServices = yield* Effect.context(); const setupCommand = yield* resolveRootCliCommand(PreviewBrowserHost.SETUP_SUBCOMMAND); @@ -2458,6 +2460,11 @@ const make = Effect.gen(function* () { ), Effect.forkScoped, ); + // The desktop keeps the tabs an agent uses focused until its turn ends, not just per command. + yield* Stream.fromSubscription(agentTurns).pipe( + Stream.runForEach((change) => desktopChannel.turn(change.threadId, change.active)), + Effect.forkScoped, + ); // Reconnect returned pages immediately so all clients keep receiving URL and title changes. yield* desktopChannel.attached.pipe( Stream.runForEach((key) => diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 6cde73143a048..f461a99186b31 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -74,6 +74,7 @@ import * as DeviceHubProxy from "./device/DeviceHubProxy.ts"; import * as PreviewManager from "./preview/Manager.ts"; import * as PortScanner from "./preview/PortScanner.ts"; import * as ServerBrowser from "./preview/ServerBrowser.ts"; +import * as PreviewAgentActivity from "./preview/PreviewAgentActivity.ts"; import * as DesktopBrowserChannel from "./preview/DesktopBrowserChannel.ts"; import * as ServerBrowserStream from "./preview/ServerBrowserStream.ts"; import * as PreviewBrowser from "./preview/PreviewBrowser.ts"; @@ -732,7 +733,12 @@ const layerMakeRoutes = Layer.mergeAll( ), // Share the service so HTTP/WS clients single-flight reads and invalidate each other's cached patches. Layer.provide(layerPullRequestService), - Layer.provide(ServerBrowser.layer.pipe(Layer.provide(DesktopBrowserChannel.layer))), + // The same activity instance RunExecutionService reports to, by layer memoization. + Layer.provide( + ServerBrowser.layer.pipe( + Layer.provide(Layer.merge(DesktopBrowserChannel.layer, PreviewAgentActivity.layer)), + ), + ), Layer.provide(PreviewBrowser.layer), Layer.provide(PreviewAutomationBroker.layer), Layer.provide(ServerSelfUpdate.layer.pipe(Layer.provide(layerDesktopAppUpdate))), diff --git a/packages/contracts/src/desktopBrowser.ts b/packages/contracts/src/desktopBrowser.ts index c70b374910079..17d256f0d8d3e 100644 --- a/packages/contracts/src/desktopBrowser.ts +++ b/packages/contracts/src/desktopBrowser.ts @@ -25,5 +25,11 @@ export const DesktopBrowserCommand = Schema.Union([ x: Schema.Finite, y: Schema.Finite, }), + /** Whether an agent turn runs on the thread, so the tabs it uses stay focused between steps. */ + Schema.Struct({ + type: Schema.Literal("turn"), + threadId: TrimmedNonEmptyString, + active: Schema.Boolean, + }), ]); export type DesktopBrowserCommand = typeof DesktopBrowserCommand.Type; From 726ceebc2343c123dd492d11b1a3e23b5dc8640c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 25 Oct 2026 07:15:25 +0000 Subject: [PATCH 06/17] fix(desktop): preview focus holds keep their owners through navigation, turn end, and screencast stops Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/preview/CdpRelay.test.ts | 12 +--- apps/desktop/src/preview/CdpRelay.ts | 6 +- .../src/preview/DesktopBrowserHost.test.ts | 70 +++++++++++++++++++ .../desktop/src/preview/DesktopBrowserHost.ts | 32 +++++---- 4 files changed, 97 insertions(+), 23 deletions(-) diff --git a/apps/desktop/src/preview/CdpRelay.test.ts b/apps/desktop/src/preview/CdpRelay.test.ts index de03ae90dc0ca..e5acd3ece936d 100644 --- a/apps/desktop/src/preview/CdpRelay.test.ts +++ b/apps/desktop/src/preview/CdpRelay.test.ts @@ -65,14 +65,8 @@ describe("CDP relay", () => { ]); pending[0]!(); await settle(); - expect(send).toHaveBeenNthCalledWith( - 1, - "Page.captureScreenshot", - {}, - undefined, - "t3-preview-page", - ); - expect(send).toHaveBeenNthCalledWith(2, "DOM.enable", {}, "iframe-session", "iframe-session"); + expect(send).toHaveBeenNthCalledWith(1, "Page.captureScreenshot", {}, undefined, undefined); + expect(send).toHaveBeenNthCalledWith(2, "DOM.enable", {}, "iframe-session", undefined); expect(written.at(-1)).toEqual({ id: 5, result: { method: "Page.captureScreenshot" }, @@ -99,7 +93,7 @@ describe("CDP relay", () => { ); await settle(); expect(send.mock.calls).toEqual([ - ["Runtime.runIfWaitingForDebugger", {}, "iframe-session", "iframe-session"], + ["Runtime.runIfWaitingForDebugger", {}, "iframe-session", undefined], ]); }); diff --git a/apps/desktop/src/preview/CdpRelay.ts b/apps/desktop/src/preview/CdpRelay.ts index 206c1d6c389a0..948bc361f5d92 100644 --- a/apps/desktop/src/preview/CdpRelay.ts +++ b/apps/desktop/src/preview/CdpRelay.ts @@ -5,7 +5,7 @@ export interface CdpRelayTarget { method: string, params: Record, sessionId: string | undefined, - /** The relay session that sent it; a detached one waits on nothing it sent. */ + /** The extra session that sent it, such as a viewer's; the main connection has none. */ owner?: string, ) => Promise; readonly sessionDetached?: (owner: string) => void; @@ -138,13 +138,15 @@ export function createCdpRelayConnection( const pageCommand = (command: CdpCommand): Promise => { const child = sessions.get(command.sessionId!) === "page" ? undefined : command.sessionId; + const owner = + child === undefined && command.sessionId !== PAGE_SESSION_ID ? command.sessionId : undefined; // Child frames/workers pause on auto-attach until resumed; only the main page can skip this command. if (command.method === "Runtime.runIfWaitingForDebugger" && child === undefined) { return Promise.resolve({}); } // Playwright asks for this on every page; the host emulates focus only while it is in use. if (command.method === "Emulation.setFocusEmulationEnabled") return Promise.resolve({}); - return target.send(command.method, command.params ?? {}, child, command.sessionId); + return target.send(command.method, command.params ?? {}, child, owner); }; return { diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 3a82b8afabf27..65ca2e5446282 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -282,6 +282,22 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect("keeps a guest focused for the screenshot a navigation lets start", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + + yield* command(host, 1, "Page.captureScreenshot"); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + debuggee.navigate(); + expect(debuggee.sent).toEqual([1, 2]); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + }), + ); + it.effect("keeps a guest the agent used focused through its turn's pauses", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; @@ -331,6 +347,28 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect("keeps a viewer's pending command focused after the agent's turn ends", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + yield* turn(true); + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Runtime.evaluate", sessionId: "t3-preview-page-1" }); + yield* settle; + yield* turn(false); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + }).pipe(Effect.scoped), + ); + it.effect("lets a guest throttle once the session waiting on its command detaches", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; @@ -387,6 +425,38 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect("keeps a guest focused while any of its screencasts runs", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + // Two viewers, or a viewer and a recording, each on its own session. + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([1, 2]); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" }); + yield* raw({ id: 5, method: "Page.stopScreencast", sessionId: "t3-preview-page-1" }); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* raw({ + id: 6, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-2" }, + }); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + it.effect("saves a server tab's download under its CDP guid where the server asked", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 2584bff83ecd5..dcecdf0b3cba9 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -49,12 +49,13 @@ interface AttachedTab { focusEmulated: boolean; focusChange: Promise | null; focusUses: number; - /** Each pending command, by the relay session waiting on it. */ + /** Each pending command, by the extra session waiting on it; the main connection's have none. */ readonly focusPending: Map; /** Used by the agent during its thread's current turn. */ turnHeld: boolean; focusReleasing: boolean; - screencasting: boolean; + /** The sessions streaming the guest; one stopping leaves the others' screencasts running. */ + readonly screencasts: Set; readonly onMessage: ( event: Electron.Event, method: string, @@ -164,7 +165,7 @@ export const make = Effect.gen(function* () { } while (uses !== tab.focusUses); tab.focusReleasing = false; // The last pending command to settle starts another idle wait. - if (tab.focusPending.size > 0 || tab.screencasting || tab.turnHeld) return; + if (tab.focusPending.size > 0 || tab.screencasts.size > 0 || tab.turnHeld) return; if (tabs.get(keyOf(tab.key)) === tab) setFocusEmulation(tab, false); }), ); @@ -175,7 +176,8 @@ export const make = Effect.gen(function* () { const pending = {}; tab.focusPending.set(pending, owner); // The agent pauses to think between steps; its tab stays focused until the turn ends. - if (agentTurns.has(tab.key.threadId)) tab.turnHeld = true; + // Viewers and recordings use extra sessions; only the main connection is the agent's. + if (owner === undefined && agentTurns.has(tab.key.threadId)) tab.turnHeld = true; tab.focusUses += 1; setFocusEmulation(tab, true); releaseFocusWhenIdle(tab); @@ -198,19 +200,24 @@ export const make = Effect.gen(function* () { const dropRelay = (tab: AttachedTab) => { tab.relay = null; - tab.screencasting = false; + tab.screencasts.clear(); tab.turnHeld = false; dropFocusHolds(tab); }; - /** A detached session, such as a closed viewer, waits on nothing it sent. */ - const dropSessionHolds = (tab: AttachedTab, owner: string) => { + const dropOwnerHolds = (tab: AttachedTab, owner: string | undefined) => { for (const [pending, sender] of tab.focusPending) { if (sender === owner) tab.focusPending.delete(pending); } releaseFocusWhenIdle(tab); }; + /** A detached session, such as a closed viewer, waits on nothing it sent. */ + const dropSessionHolds = (tab: AttachedTab, owner: string) => { + tab.screencasts.delete(owner); + dropOwnerHolds(tab, owner); + }; + /** * A command the server stopped waiting on, such as a timed-out screenshot of a hidden * guest, may never settle; the end of the turn that sent it ends its hold too. @@ -220,7 +227,7 @@ export const make = Effect.gen(function* () { for (const tab of tabs.values()) { if (tab.key.threadId !== threadId) continue; tab.turnHeld = false; - dropFocusHolds(tab); + dropOwnerHolds(tab, undefined); } }; @@ -234,8 +241,8 @@ export const make = Effect.gen(function* () { const relay: CdpRelayConnection = createCdpRelayConnection( { send: (method, params, sessionId, owner) => { - if (method === "Page.startScreencast") tab.screencasting = true; - else if (method === "Page.stopScreencast") tab.screencasting = false; + if (method === "Page.startScreencast") tab.screencasts.add(owner); + else if (method === "Page.stopScreencast") tab.screencasts.delete(owner); // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. const forward = () => { const change = tab.focusChange; @@ -310,7 +317,7 @@ export const make = Effect.gen(function* () { focusPending: new Map(), turnHeld: false, focusReleasing: false, - screencasting: false, + screencasts: new Set(), onMessage: (_event, method, params, sessionId) => { if (method === "Browser.downloadWillBegin") { const guid = (params as { guid?: unknown } | undefined)?.guid; @@ -321,8 +328,9 @@ export const make = Effect.gen(function* () { // A screenshot pending when the page loads a new document, a reload included, can // stay unanswered. A crashed page lands here once it reloads, well after its renderer died. onNewDocument: () => { - if (tab.screenshotTurn !== null) nextScreenshot(tab); + // Before the next screenshot starts, so its own hold survives. dropFocusHolds(tab); + if (tab.screenshotTurn !== null) nextScreenshot(tab); }, }; tabs.set(id, tab); From 15d6e0dbe6f793c5e08c65dba0ff8857e302835b Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 25 Oct 2026 07:21:24 +0000 Subject: [PATCH 07/17] fix(desktop): preview focus holds follow the agent's own use and the current screenshot A screenshot that waited on focus emulation could still be sent after a navigation had started the next one, putting two captures in flight. It now checks it still owns the screenshot turn, relay and document right before sending. A turn now holds focus only on tabs the agent drove with input, navigation or evaluation, not Playwright housekeeping, and a turn whose start arrives after its first command still holds the tab that command used. Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 72 ++++++++++++++++++- .../desktop/src/preview/DesktopBrowserHost.ts | 51 ++++++++----- 2 files changed, 105 insertions(+), 18 deletions(-) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 65ca2e5446282..a84280699bd92 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -31,9 +31,10 @@ const command = ( }), ); -const makeDebuggee = () => { +const makeDebuggee = (options: { readonly deferFocus?: boolean } = {}) => { const emitter = new NodeEvents.EventEmitter(); const pending: Array<() => void> = []; + const focusChanges: Array<() => void> = []; const sent: Array = []; const sendWaiters: Array<() => void> = []; const focusEmulation: Array = []; @@ -46,7 +47,8 @@ const makeDebuggee = () => { } if (method === "Emulation.setFocusEmulationEnabled") { focusEmulation.push(params?.enabled === true); - resolve({}); + if (options.deferFocus) focusChanges.push(() => resolve({})); + else resolve({}); return; } sent.push(params?.id); @@ -72,6 +74,7 @@ const makeDebuggee = () => { nextSend: () => new Promise((resolve) => sendWaiters.push(resolve)), release: () => pending.splice(0).forEach((resolve) => resolve()), releaseFirst: () => pending.shift()?.(), + settleFocus: () => focusChanges.splice(0).forEach((resolve) => resolve()), }; }; @@ -298,6 +301,71 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect("never sends a screenshot a navigation superseded while focus was changing", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee({ deferFocus: true }); + host.attach(key, debuggee.tab); + + yield* command(host, 1, "Page.captureScreenshot"); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + debuggee.navigate(); + debuggee.settleFocus(); + yield* settle; + // Two captures in flight would undo the one-at-a-time guard. + expect(debuggee.sent).toEqual([2]); + }), + ); + + it.effect("keeps a guest focused when its turn starts after the agent's first command", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + yield* command(host, 1, "Input.dispatchKeyEvent"); + yield* settle; + yield* turn(true); + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* turn(false); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }), + ); + + it.effect("lets a guest throttle through a turn that only sent it housekeeping", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + yield* host.handleCommandLine( + encodeJson({ type: "turn", threadId: key.threadId, active: true }), + ); + + // Playwright re-enabling a domain after a reload is not the agent using the tab. + yield* command(host, 1, "Runtime.enable"); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + + yield* command(host, 2, "Runtime.evaluate"); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false, true]); + }), + ); + it.effect("keeps a guest the agent used focused through its turn's pauses", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index dcecdf0b3cba9..c11d06550818e 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -22,6 +22,8 @@ const decodeCommand = Schema.decodeUnknownOption(Schema.fromJsonString(DesktopBr const lineEncoder = new TextEncoder(); const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; +/** The same commands as above, by method, for the main connection's sends. */ +const AGENT_INPUT_METHOD = /^(?:Input\.|Page\.navigate$|Runtime\.(?:evaluate|callFunctionOn)$)/; const AGENT_DOWNLOAD_WINDOW_MS = 5_000; const FOCUS_IDLE = "5 seconds"; @@ -51,8 +53,11 @@ interface AttachedTab { focusUses: number; /** Each pending command, by the extra session waiting on it; the main connection's have none. */ readonly focusPending: Map; - /** Used by the agent during its thread's current turn. */ - turnHeld: boolean; + /** + * Driven by the agent since its thread's last turn ended or the guest last throttled. A + * turn's start can reach us after its first command, so this holds focus once it does. + */ + agentUsed: boolean; focusReleasing: boolean; /** The sessions streaming the guest; one stopping leaves the others' screencasts running. */ readonly screencasts: Set; @@ -111,7 +116,8 @@ export const make = Effect.gen(function* () { const takeScreenshot = ( tab: AttachedTab, relay: CdpRelayConnection, - capture: () => Promise, + /** Sends only while `current` holds, which focus emulation may outlast. */ + capture: (current: () => boolean) => Promise, ) => new Promise((resolve, reject) => { const start = () => { @@ -122,7 +128,7 @@ export const make = Effect.gen(function* () { } const turn = {}; tab.screenshotTurn = turn; - capture() + capture(() => tab.screenshotTurn === turn && tab.relay === relay) .finally(() => { if (tab.screenshotTurn === turn) nextScreenshot(tab); }) @@ -165,19 +171,27 @@ export const make = Effect.gen(function* () { } while (uses !== tab.focusUses); tab.focusReleasing = false; // The last pending command to settle starts another idle wait. - if (tab.focusPending.size > 0 || tab.screencasts.size > 0 || tab.turnHeld) return; + if (tab.focusPending.size > 0 || tab.screencasts.size > 0) return; + // The agent pauses to think between steps; its tab stays focused until the turn ends. + if (tab.agentUsed && agentTurns.has(tab.key.threadId)) return; + tab.agentUsed = false; if (tabs.get(keyOf(tab.key)) === tab) setFocusEmulation(tab, false); }), ); }; /** A command may wait on the page's next frame, so the guest stays focused until it settles. */ - const holdFocus = (tab: AttachedTab, owner: string | undefined, work: () => Promise) => { + const holdFocus = ( + tab: AttachedTab, + method: string, + owner: string | undefined, + work: () => Promise, + ) => { const pending = {}; tab.focusPending.set(pending, owner); - // The agent pauses to think between steps; its tab stays focused until the turn ends. - // Viewers and recordings use extra sessions; only the main connection is the agent's. - if (owner === undefined && agentTurns.has(tab.key.threadId)) tab.turnHeld = true; + // Viewers and recordings use extra sessions; only the main connection is the agent's, + // and Playwright's own housekeeping on it is not the agent using the tab. + if (owner === undefined && AGENT_INPUT_METHOD.test(method)) tab.agentUsed = true; tab.focusUses += 1; setFocusEmulation(tab, true); releaseFocusWhenIdle(tab); @@ -201,7 +215,7 @@ export const make = Effect.gen(function* () { const dropRelay = (tab: AttachedTab) => { tab.relay = null; tab.screencasts.clear(); - tab.turnHeld = false; + tab.agentUsed = false; dropFocusHolds(tab); }; @@ -226,7 +240,7 @@ export const make = Effect.gen(function* () { agentTurns.delete(threadId); for (const tab of tabs.values()) { if (tab.key.threadId !== threadId) continue; - tab.turnHeld = false; + tab.agentUsed = false; dropOwnerHolds(tab, undefined); } }; @@ -244,15 +258,20 @@ export const make = Effect.gen(function* () { if (method === "Page.startScreencast") tab.screencasts.add(owner); else if (method === "Page.stopScreencast") tab.screencasts.delete(owner); // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. - const forward = () => { + const forward = (current?: () => boolean) => { const change = tab.focusChange; - const run = () => send(method, params, sessionId); + const run = () => + current === undefined || current() + ? send(method, params, sessionId) + : Promise.reject(new Error("The page this screenshot was for has moved on.")); return change === null ? run() : change.then(run); }; // A screenshot holds focus only once its turn starts, not while queued behind another. return method === "Page.captureScreenshot" - ? takeScreenshot(tab, relay, () => holdFocus(tab, owner, forward)) - : holdFocus(tab, owner, forward); + ? takeScreenshot(tab, relay, (current) => + holdFocus(tab, method, owner, () => forward(current)), + ) + : holdFocus(tab, method, owner, forward); }, sessionDetached: (owner) => dropSessionHolds(tab, owner), targetId: () => @@ -315,7 +334,7 @@ export const make = Effect.gen(function* () { focusChange: null, focusUses: 0, focusPending: new Map(), - turnHeld: false, + agentUsed: false, focusReleasing: false, screencasts: new Set(), onMessage: (_event, method, params, sessionId) => { From 78c92138081447339d6b0fe6adf1bb98965b0437 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 25 Oct 2026 07:34:31 +0000 Subject: [PATCH 08/17] fix(desktop): a hidden preview guest paints while the agent captures it Electron composites no offscreen guest, so a screenshot or screencast of a hidden preview tab never gets a frame and stays pending, keeping the guest captured at 20Hz. The host now reports while a screenshot it is taking or a screencast is pending, the Manager carries it on the tab state as capturing, and the client holds the existing surface activity lease, which paints the guest behind the app until the capture ends or its hold is dropped. Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 112 ++++++++++++++++++ .../desktop/src/preview/DesktopBrowserHost.ts | 43 ++++++- apps/desktop/src/preview/Manager.test.ts | 51 ++++++++ apps/desktop/src/preview/Manager.ts | 41 +++++-- .../preview/usePreviewBridge.test.ts | 25 +++- .../components/preview/usePreviewBridge.ts | 30 ++++- packages/contracts/src/ipc.ts | 2 + 7 files changed, 288 insertions(+), 16 deletions(-) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index a84280699bd92..957317d890314 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -108,6 +108,18 @@ const watchReplies = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) => return replies; }); +/** Whether the client should paint the guest, each time that changes. */ +const watchPainting = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) => + Effect.gen(function* () { + const painting: Array = []; + yield* host.painting.pipe( + Stream.runForEach(({ active }) => Effect.sync(() => painting.push(active))), + Effect.forkScoped, + ); + yield* Effect.yieldNow; + return painting; + }); + describe("DesktopBrowserHost", () => { it.effect("announces tabs already attached to a backend that starts later", () => Effect.gen(function* () { @@ -525,6 +537,106 @@ describe("DesktopBrowserHost", () => { }).pipe(Effect.scoped), ); + it.effect("paints a hidden guest only while the screenshot it is taking is pending", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const painting = yield* watchPainting(host); + + yield* command(host, 1, "DOM.enable"); + yield* settle; + expect(painting).toEqual([]); + + yield* command(host, 2, "Page.captureScreenshot"); + yield* command(host, 3, "Page.captureScreenshot"); + yield* settle; + expect(painting).toEqual([true]); + + // The first answers once the guest paints; the queued one then takes its turn. + debuggee.release(); + yield* settle; + expect(debuggee.sent).toEqual([1, 2, 3]); + debuggee.release(); + yield* settle; + expect(painting).toEqual([true, false, true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("stops painting for a screenshot Chromium never answers", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const painting = yield* watchPainting(host); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + // The server gave up on it, and the agent's turn ends. + yield* turn(true); + yield* command(host, 1, "Page.captureScreenshot"); + yield* settle; + yield* turn(false); + yield* settle; + expect(painting).toEqual([true, false]); + + // The page loads a new document, which also frees the screenshot turn. + debuggee.navigate(); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + debuggee.navigate(); + yield* settle; + expect(painting).toEqual([true, false, true, false]); + + // The server releases the relay. + yield* command(host, 3, "Page.captureScreenshot"); + yield* settle; + yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); + yield* settle; + expect(painting).toEqual([true, false, true, false, true, false]); + + // The guest detaches; a replacement starts with no screenshot pending. + const replacement = makeDebuggee(); + host.attach(key, replacement.tab); + yield* command(host, 4, "Page.captureScreenshot"); + yield* settle; + host.detach(key); + yield* settle; + expect(painting).toEqual([true, false, true, false, true, false, true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("paints a streamed guest until its last screencast ends", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const painting = yield* watchPainting(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([1, 2]); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" }); + yield* raw({ id: 5, method: "Page.stopScreencast", sessionId: "t3-preview-page-1" }); + yield* settle; + debuggee.release(); + yield* settle; + expect(painting).toEqual([true]); + + yield* raw({ + id: 6, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-2" }, + }); + yield* settle; + expect(painting).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + it.effect("saves a server tab's download under its CDP guid where the server asked", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index c11d06550818e..818a6c8819b1f 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -52,7 +52,12 @@ interface AttachedTab { focusChange: Promise | null; focusUses: number; /** Each pending command, by the extra session waiting on it; the main connection's have none. */ - readonly focusPending: Map; + readonly focusPending: Map< + object, + { readonly owner: string | undefined; readonly paints: boolean } + >; + /** Whether the client keeps the guest painting, as published on `painting`. */ + painting: boolean; /** * Driven by the agent since its thread's last turn ended or the guest last throttled. A * turn's start can reach us after its first command, so this holds focus once it does. @@ -85,6 +90,11 @@ export class DesktopBrowserHost extends Context.Service< readonly x: number; readonly y: number; }>; + /** Whether a guest needs its frames painted while the panel hides it. */ + readonly painting: Stream.Stream<{ + readonly key: DesktopBrowserTabKey; + readonly active: boolean; + }>; } >()("@t3tools/desktop/preview/DesktopBrowserHost") {} @@ -96,6 +106,10 @@ export const make = Effect.gen(function* () { readonly x: number; readonly y: number; }>(16); + const painting = yield* PubSub.unbounded<{ + readonly key: DesktopBrowserTabKey; + readonly active: boolean; + }>(); const runFork = Effect.runForkWith(yield* Effect.context()); const tabs = new Map(); /** Threads whose agent turn runs, by the server's account. */ @@ -180,6 +194,20 @@ export const make = Effect.gen(function* () { ); }; + /** + * Electron composites no offscreen guest, so its screenshot or screencast waits for a frame + * that never comes. The client paints the guest behind the app while one is pending; the + * capture then finishes on its own, and the guest goes back offscreen once none is. + */ + const syncPainting = (tab: AttachedTab) => { + const active = + tabs.get(keyOf(tab.key)) === tab && + (tab.screencasts.size > 0 || [...tab.focusPending.values()].some((hold) => hold.paints)); + if (tab.painting === active) return; + tab.painting = active; + runFork(PubSub.publish(painting, { key: tab.key, active })); + }; + /** A command may wait on the page's next frame, so the guest stays focused until it settles. */ const holdFocus = ( tab: AttachedTab, @@ -188,17 +216,19 @@ export const make = Effect.gen(function* () { work: () => Promise, ) => { const pending = {}; - tab.focusPending.set(pending, owner); + tab.focusPending.set(pending, { owner, paints: method === "Page.captureScreenshot" }); // Viewers and recordings use extra sessions; only the main connection is the agent's, // and Playwright's own housekeeping on it is not the agent using the tab. if (owner === undefined && AGENT_INPUT_METHOD.test(method)) tab.agentUsed = true; tab.focusUses += 1; setFocusEmulation(tab, true); releaseFocusWhenIdle(tab); + syncPainting(tab); return work().finally(() => { if (!tab.focusPending.delete(pending)) return; tab.focusUses += 1; releaseFocusWhenIdle(tab); + syncPainting(tab); }); }; @@ -210,6 +240,7 @@ export const make = Effect.gen(function* () { const dropFocusHolds = (tab: AttachedTab) => { tab.focusPending.clear(); releaseFocusWhenIdle(tab); + syncPainting(tab); }; const dropRelay = (tab: AttachedTab) => { @@ -220,10 +251,11 @@ export const make = Effect.gen(function* () { }; const dropOwnerHolds = (tab: AttachedTab, owner: string | undefined) => { - for (const [pending, sender] of tab.focusPending) { - if (sender === owner) tab.focusPending.delete(pending); + for (const [pending, hold] of tab.focusPending) { + if (hold.owner === owner) tab.focusPending.delete(pending); } releaseFocusWhenIdle(tab); + syncPainting(tab); }; /** A detached session, such as a closed viewer, waits on nothing it sent. */ @@ -314,6 +346,7 @@ export const make = Effect.gen(function* () { tab.relay = null; tab.debuggee.debugger.off("message", tab.onMessage); tab.debuggee.webContents.off("did-navigate", tab.onNewDocument); + syncPainting(tab); emit({ type: "detached", ...key }); }; @@ -334,6 +367,7 @@ export const make = Effect.gen(function* () { focusChange: null, focusUses: 0, focusPending: new Map(), + painting: false, agentUsed: false, focusReleasing: false, screencasts: new Set(), @@ -402,6 +436,7 @@ export const make = Effect.gen(function* () { return DesktopBrowserHost.of({ pointers: Stream.fromPubSub(pointers), + painting: Stream.fromPubSub(painting), // Subscribes before announcing, so no attach falls between the two. events: Stream.unwrap( Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 3599e7d3cef28..2146e6cafe1b5 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1713,6 +1713,57 @@ describe("PreviewManager", () => { }).pipe(Effect.provide(managerLayer()), Effect.scoped), ); + effectIt.effect("asks the client to paint a server tab while the agent captures it", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.DesktopBrowserHost; + const serverTab = { threadId: "thread-1", tabId: "server-tab-1" }; + const contents = Object.assign( + makeTestPreviewWebContents(async () => { + throw new Error("unused"); + }, 42), + { isDevToolsOpened: () => false, getUserAgent: () => "Electron" }, + ); + const screenshot = Promise.withResolvers(); + contents.debugger.sendCommand = vi.fn((method: string) => + method === "Page.captureScreenshot" ? screenshot.promise : Promise.resolve(undefined), + ) as unknown as typeof contents.debugger.sendCommand; + fromId.mockReturnValue(contents); + const manager = yield* PreviewManager.PreviewManager; + const capturing = yield* Queue.unbounded(); + yield* manager.subscribeStateChanges((_tabId, state) => + Queue.offer(capturing, state.capturing).pipe(Effect.asVoid), + ); + const attached = yield* Deferred.make(); + yield* host.events.pipe( + Stream.runForEach((line) => + new TextDecoder().decode(line).includes('"type":"attached"') + ? Deferred.succeed(attached, undefined) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* manager.createTab("tab_1", { serverTab }); + yield* manager.registerWebview("tab_1", 42); + yield* Deferred.await(attached); + yield* Queue.clear(capturing); + + yield* host.handleCommandLine( + JSON.stringify({ + type: "cdp", + ...serverTab, + message: JSON.stringify({ + id: 1, + method: "Page.captureScreenshot", + sessionId: "t3-preview-page", + }), + }), + ); + expect(yield* Queue.take(capturing)).toBe(true); + screenshot.resolve({ data: "" }); + expect(yield* Queue.take(capturing)).toBe(false); + }).pipe(Effect.provide(managerLayer()), Effect.scoped), + ); + effectIt.effect("discards a screenshot that resolves after its guest is replaced", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index b9e0192ca8a7d..3e3f6f3d7bf5d 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -102,6 +102,7 @@ export interface PreviewTabState { audioMuted: boolean; audible: boolean; controller: "human" | "agent" | "none"; + capturing: boolean; favicon?: DesktopPreviewFavicon; serverTab?: { readonly threadId: string; readonly tabId: string }; updatedAt: string; @@ -1652,6 +1653,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function audioMuted: false, audible: false, controller: "none", + capturing: false, updatedAt, }; return [ @@ -1725,6 +1727,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function audioMuted: false, audible: false, controller: "none", + capturing: false, updatedAt, }; yield* emit(tabId, closed); @@ -1946,6 +1949,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function audioMuted: current?.audioMuted ?? false, audible: current?.audible ?? false, controller: current?.controller ?? "none", + capturing: current?.capturing ?? false, ...(current?.favicon ? { favicon: current.favicon } : {}), updatedAt, }; @@ -2299,7 +2303,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return; } if (afterAttach.serverTab) { - yield* listenForAgentPointers; + yield* listenToBrowserHost; browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger }); } if (afterAttach.colorScheme !== "system") { @@ -3239,14 +3243,33 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { discard: true }, ); }); - let pointersStarted = false; - const listenForAgentPointers = Effect.suspend(() => { - if (pointersStarted) return Effect.void; - pointersStarted = true; - return browserHost.pointers.pipe( - Stream.runForEach(emitAgentPointer), - Effect.forkIn(parentScope), - Effect.asVoid, + const updateServerTabCapturing = Effect.fn("PreviewManager.updateServerTabCapturing")( + function* (painting: { + readonly key: { readonly threadId: string; readonly tabId: string }; + readonly active: boolean; + }) { + const tab = [...(yield* SynchronizedRef.get(tabsRef)).values()].find( + (candidate) => + candidate.serverTab?.threadId === painting.key.threadId && + candidate.serverTab.tabId === painting.key.tabId, + ); + if (tab && tab.capturing !== painting.active) { + yield* update(tab.tabId, { capturing: painting.active }); + } + }, + ); + let browserHostStarted = false; + // Subscribes before the attach that follows, so no event falls between the two. + const listenToBrowserHost = Effect.suspend(() => { + if (browserHostStarted) return Effect.void; + browserHostStarted = true; + const listen = Effect.forkIn(parentScope, { startImmediately: true }); + return Effect.all( + [ + browserHost.pointers.pipe(Stream.runForEach(emitAgentPointer), listen), + browserHost.painting.pipe(Stream.runForEach(updateServerTabCapturing), listen), + ], + { discard: true }, ); }); diff --git a/apps/web/src/components/preview/usePreviewBridge.test.ts b/apps/web/src/components/preview/usePreviewBridge.test.ts index 14acf0d69609e..736ac7d743c32 100644 --- a/apps/web/src/components/preview/usePreviewBridge.test.ts +++ b/apps/web/src/components/preview/usePreviewBridge.test.ts @@ -1,7 +1,9 @@ import type { DesktopPreviewTabState } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; -import { projectDesktopState } from "./usePreviewBridge"; +import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; + +import { createCaptureActivity, projectDesktopState } from "./usePreviewBridge"; const favicon = { dataUrl: "data:image/png;base64,AAAA", @@ -22,6 +24,7 @@ function state(navStatus: DesktopPreviewTabState["navStatus"]): DesktopPreviewTa audioMuted: false, audible: false, controller: "none", + capturing: false, favicon, updatedAt: "2026-08-09T00:00:00.000Z", }; @@ -48,3 +51,23 @@ describe("projectDesktopState", () => { expect(projectDesktopState(state({ kind: "Idle" })).favicon).toBeNull(); }); }); + +describe("createCaptureActivity", () => { + it("keeps the guest painting while the host captures it, alongside other activity", () => { + const activity = () => useBrowserSurfaceStore.getState().activityByTabId["tab-1"]; + const recording = useBrowserSurfaceStore.getState().acquireActivity("tab-1"); + const capture = createCaptureActivity("tab-1"); + + capture.update(true); + capture.update(true); + expect(activity()).toBe(2); + capture.update(false); + expect(activity()).toBe(1); + capture.update(true); + capture.release(); + capture.release(); + expect(activity()).toBe(1); + recording(); + expect(activity()).toBeUndefined(); + }); +}); diff --git a/apps/web/src/components/preview/usePreviewBridge.ts b/apps/web/src/components/preview/usePreviewBridge.ts index a58382d4f83e8..91fc2041e2e87 100644 --- a/apps/web/src/components/preview/usePreviewBridge.ts +++ b/apps/web/src/components/preview/usePreviewBridge.ts @@ -8,6 +8,7 @@ import { useEffect, useEffectEvent, useRef } from "react"; import { recordFaviconForThread } from "~/browserFaviconStore"; import { useBrowserPointerStore } from "~/browser/browserPointerStore"; +import { acquireBrowserSurfaceActivity } from "~/browser/browserSurfaceStore"; import { applyPreviewDesktopState, type DesktopPreviewOverlay } from "~/previewStateStore"; import { previewEnvironment } from "~/state/preview"; import { useAtomCommand } from "~/state/use-atom-command"; @@ -22,6 +23,22 @@ function originOf(url: string): string | null { } } +/** + * Keeps a hidden guest painting behind the app while the desktop host captures it; Electron + * composites no offscreen guest, so its screenshot or stream would otherwise never finish. + */ +export function createCaptureActivity(tabId: string) { + let release: (() => void) | null = null; + const update = (capturing: boolean) => { + if (capturing && !release) release = acquireBrowserSurfaceActivity(tabId); + else if (!capturing && release) { + release(); + release = null; + } + }; + return { update, release: () => update(false) }; +} + /** ElectronBrowserHost owns webview lifetime. */ export function usePreviewBridge(input: { threadRef: ScopedThreadRef; @@ -38,6 +55,7 @@ export function usePreviewBridge(input: { const lastReportedUrl = useRef(null); const lastReportedKind = useRef(null); const lastDesktopNavStatus = useRef(null); + const captureActivity = useRef | null>(null); const handleStateChange = useEffectEvent( (changedTabId: string, state: DesktopPreviewTabState): void => { if (changedTabId !== runtimeTabId) return; @@ -45,6 +63,7 @@ export function usePreviewBridge(input: { clearBrowserPointer(runtimeTabId); } lastDesktopNavStatus.current = state.navStatus; + captureActivity.current?.update(state.capturing); applyPreviewDesktopState(stableThreadRef, tabId, projectDesktopState(state)); if (state.favicon) { recordFaviconForThread(stableThreadRef, state.favicon); @@ -70,10 +89,17 @@ export function usePreviewBridge(input: { lastReportedUrl.current = null; lastReportedKind.current = null; lastDesktopNavStatus.current = null; - return previewBridge.onStateChange(handleStateChange); + const activity = createCaptureActivity(runtimeTabId); + captureActivity.current = activity; + const unsubscribe = previewBridge.onStateChange(handleStateChange); + return () => { + unsubscribe(); + activity.release(); + if (captureActivity.current === activity) captureActivity.current = null; + }; }, [ - // oxlint-disable-next-line react/exhaustive-effect-dependencies -- A different tab or thread starts its status reports and pointer tracking from scratch. runtimeTabId, + // oxlint-disable-next-line react/exhaustive-effect-dependencies -- A different tab or thread starts its status reports and pointer tracking from scratch. stableThreadRef, tabId, ]); diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 1a91b8a259441..34d4162081751 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -590,6 +590,8 @@ export interface DesktopPreviewTabState { /** Chromium reports playback even when audioMuted is true. */ audible: boolean; controller: "human" | "agent" | "none"; + /** A screenshot or stream of the guest is pending, so the client keeps it painting while hidden. */ + capturing: boolean; favicon?: DesktopPreviewFavicon; updatedAt: string; } From c8e04b8648ef38af02d8ad836d1e34761d946975 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 25 Oct 2026 08:14:34 +0000 Subject: [PATCH 09/17] fix(desktop): a navigated preview tab keeps the server tab it answers for Navigating rebuilt the tab's state without its server tab, so once a user loaded a URL the Manager could no longer match the host's painting, pointer, or detach events to it. A hidden guest the agent then captured never painted, its screenshot timed out, and it stayed captured at 20Hz. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/preview/Manager.test.ts | 104 ++++++++++++----------- apps/desktop/src/preview/Manager.ts | 1 + 2 files changed, 57 insertions(+), 48 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 2146e6cafe1b5..f19079e46bce4 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1713,55 +1713,63 @@ describe("PreviewManager", () => { }).pipe(Effect.provide(managerLayer()), Effect.scoped), ); - effectIt.effect("asks the client to paint a server tab while the agent captures it", () => - Effect.gen(function* () { - const host = yield* DesktopBrowserHost.DesktopBrowserHost; - const serverTab = { threadId: "thread-1", tabId: "server-tab-1" }; - const contents = Object.assign( - makeTestPreviewWebContents(async () => { - throw new Error("unused"); - }, 42), - { isDevToolsOpened: () => false, getUserAgent: () => "Electron" }, - ); - const screenshot = Promise.withResolvers(); - contents.debugger.sendCommand = vi.fn((method: string) => - method === "Page.captureScreenshot" ? screenshot.promise : Promise.resolve(undefined), - ) as unknown as typeof contents.debugger.sendCommand; - fromId.mockReturnValue(contents); - const manager = yield* PreviewManager.PreviewManager; - const capturing = yield* Queue.unbounded(); - yield* manager.subscribeStateChanges((_tabId, state) => - Queue.offer(capturing, state.capturing).pipe(Effect.asVoid), - ); - const attached = yield* Deferred.make(); - yield* host.events.pipe( - Stream.runForEach((line) => - new TextDecoder().decode(line).includes('"type":"attached"') - ? Deferred.succeed(attached, undefined) - : Effect.void, - ), - Effect.forkScoped, - ); - yield* manager.createTab("tab_1", { serverTab }); - yield* manager.registerWebview("tab_1", 42); - yield* Deferred.await(attached); - yield* Queue.clear(capturing); - - yield* host.handleCommandLine( - JSON.stringify({ - type: "cdp", - ...serverTab, - message: JSON.stringify({ - id: 1, - method: "Page.captureScreenshot", - sessionId: "t3-preview-page", + effectIt.effect( + "asks the client to paint a navigated server tab while the agent captures it", + () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.DesktopBrowserHost; + const serverTab = { threadId: "thread-1", tabId: "server-tab-1" }; + const contents = Object.assign( + makeTestPreviewWebContents(async () => { + throw new Error("unused"); + }, 42), + { + isDevToolsOpened: () => false, + getUserAgent: () => "Electron", + loadURL: vi.fn(async () => undefined), + }, + ); + const screenshot = Promise.withResolvers(); + contents.debugger.sendCommand = vi.fn((method: string) => + method === "Page.captureScreenshot" ? screenshot.promise : Promise.resolve(undefined), + ) as unknown as typeof contents.debugger.sendCommand; + fromId.mockReturnValue(contents); + const manager = yield* PreviewManager.PreviewManager; + const capturing = yield* Queue.unbounded(); + yield* manager.subscribeStateChanges((_tabId, state) => + Queue.offer(capturing, state.capturing).pipe(Effect.asVoid), + ); + const attached = yield* Deferred.make(); + yield* host.events.pipe( + Stream.runForEach((line) => + new TextDecoder().decode(line).includes('"type":"attached"') + ? Deferred.succeed(attached, undefined) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* manager.createTab("tab_1", { serverTab }); + yield* manager.registerWebview("tab_1", 42); + yield* Deferred.await(attached); + // Navigating rebuilds the tab's state, which must keep the server tab it answers for. + yield* manager.navigate("tab_1", "https://example.com/next"); + yield* Queue.clear(capturing); + + yield* host.handleCommandLine( + JSON.stringify({ + type: "cdp", + ...serverTab, + message: JSON.stringify({ + id: 1, + method: "Page.captureScreenshot", + sessionId: "t3-preview-page", + }), }), - }), - ); - expect(yield* Queue.take(capturing)).toBe(true); - screenshot.resolve({ data: "" }); - expect(yield* Queue.take(capturing)).toBe(false); - }).pipe(Effect.provide(managerLayer()), Effect.scoped), + ); + expect(yield* Queue.take(capturing)).toBe(true); + screenshot.resolve({ data: "" }); + expect(yield* Queue.take(capturing)).toBe(false); + }).pipe(Effect.provide(managerLayer()), Effect.scoped), ); effectIt.effect("discards a screenshot that resolves after its guest is replaced", () => diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 3e3f6f3d7bf5d..d0221e9be0e6e 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -1951,6 +1951,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function controller: current?.controller ?? "none", capturing: current?.capturing ?? false, ...(current?.favicon ? { favicon: current.favicon } : {}), + ...(current?.serverTab ? { serverTab: current.serverTab } : {}), updatedAt, }; return [ From d13a9f72298d362239b6f199d33435ec0f4af9bc Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sun, 25 Oct 2026 17:55:17 -0700 Subject: [PATCH 10/17] fix(desktop): a detached viewer's queued screenshot never starts A screenshot a viewer queued behind another could still start after the viewer detached, holding focus and painting with nobody waiting. The bridge effect now keys on thread identifiers, not a per-render object. Co-Authored-By: Julius Marminge Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 30 +++++++++++++++++++ .../desktop/src/preview/DesktopBrowserHost.ts | 10 +++++-- .../components/preview/usePreviewBridge.ts | 3 +- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 957317d890314..caad7e9678814 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -476,6 +476,36 @@ describe("DesktopBrowserHost", () => { }).pipe(Effect.scoped), ); + it.effect("never sends a screenshot queued by a session that has since detached", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* command(host, 2, "Page.captureScreenshot"); + yield* raw({ id: 3, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" }); + yield* raw({ + id: 4, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + yield* settle; + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(4); + expect(yield* Queue.take(replies)).toBe(2); + yield* settle; + expect(debuggee.sent).toEqual([2]); + + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + it.effect("keeps a streamed guest focused until its screencast ends", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 818a6c8819b1f..613a4d686a5e9 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -284,6 +284,7 @@ export const make = Effect.gen(function* () { sessionId === undefined ? sendPageCommand(webContents, debuggee, method, params) : debuggee.sendCommand(method, params, sessionId); + const detachedOwners = new Set(); const relay: CdpRelayConnection = createCdpRelayConnection( { send: (method, params, sessionId, owner) => { @@ -301,11 +302,16 @@ export const make = Effect.gen(function* () { // A screenshot holds focus only once its turn starts, not while queued behind another. return method === "Page.captureScreenshot" ? takeScreenshot(tab, relay, (current) => - holdFocus(tab, method, owner, () => forward(current)), + owner !== undefined && detachedOwners.has(owner) + ? Promise.reject(new Error("The session that asked for this screenshot closed.")) + : holdFocus(tab, method, owner, () => forward(current)), ) : holdFocus(tab, method, owner, forward); }, - sessionDetached: (owner) => dropSessionHolds(tab, owner), + sessionDetached: (owner) => { + detachedOwners.add(owner); + dropSessionHolds(tab, owner); + }, targetId: () => debuggee .sendCommand("Target.getTargetInfo") diff --git a/apps/web/src/components/preview/usePreviewBridge.ts b/apps/web/src/components/preview/usePreviewBridge.ts index 91fc2041e2e87..69c6be2046fea 100644 --- a/apps/web/src/components/preview/usePreviewBridge.ts +++ b/apps/web/src/components/preview/usePreviewBridge.ts @@ -100,7 +100,8 @@ export function usePreviewBridge(input: { }, [ runtimeTabId, // oxlint-disable-next-line react/exhaustive-effect-dependencies -- A different tab or thread starts its status reports and pointer tracking from scratch. - stableThreadRef, + environmentId, + threadId, tabId, ]); } From c379f99eddb3ebbc712329dff1534f458008dbed Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sun, 25 Oct 2026 18:13:01 -0700 Subject: [PATCH 11/17] fix(desktop): preview focus holds end with the agent's own session and never strand a screenshot The server sends evaluations and snapshots on its own extra CDP session, so turn end now drops every hold on the thread's tabs and any agent input marks the tab used. A screenshot queued behind one abandoned mid-capture now keeps the guest painting so the first can finish. Screencasts count only on page sessions, so a late one on a detached session cannot pin focus. Co-Authored-By: Julius Marminge Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 80 ++++++++++++++++++- .../desktop/src/preview/DesktopBrowserHost.ts | 52 +++++++----- .../src/preview/DesktopBrowserChannel.ts | 2 +- 3 files changed, 111 insertions(+), 23 deletions(-) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index caad7e9678814..ae258ea0538e7 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -427,7 +427,7 @@ describe("DesktopBrowserHost", () => { }), ); - it.effect("keeps a viewer's pending command focused after the agent's turn ends", () => + it.effect("ends the agent's own session's holds with its turn, but not a viewer's stream", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; const debuggee = makeDebuggee(); @@ -438,6 +438,7 @@ describe("DesktopBrowserHost", () => { const turn = (active: boolean) => host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + // The server's own CDP session, evaluating a promise that never settles. yield* turn(true); yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); expect(yield* Queue.take(replies)).toBe(1); @@ -445,7 +446,80 @@ describe("DesktopBrowserHost", () => { yield* settle; yield* turn(false); yield* TestClock.adjust("1 minute"); - expect(debuggee.focusEmulation).toEqual([true]); + expect(debuggee.focusEmulation).toEqual([true, false]); + + // A viewer streaming the guest keeps it focused through a turn's end. + yield* turn(true); + yield* raw({ id: 3, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(3); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" }); + yield* settle; + yield* turn(false); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false, true]); + }).pipe(Effect.scoped), + ); + + it.effect("paints for a screenshot queued behind one its viewer abandoned mid-capture", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const painting = yield* watchPainting(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" }); + yield* settle; + yield* raw({ + id: 3, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + expect(yield* Queue.take(replies)).toBe(3); + expect(painting).toEqual([true, false]); + + // The agent's screenshot waits its turn; painting lets the abandoned one finish first. + yield* command(host, 4, "Page.captureScreenshot"); + yield* settle; + expect(painting).toEqual([true, false, true]); + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(2); + yield* settle; + expect(debuggee.sent).toEqual([undefined, 4]); + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(4); + yield* settle; + expect(painting).toEqual([true, false, true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("ignores a screencast that arrives on a session after it detached", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ + id: 2, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + expect(yield* Queue.take(replies)).toBe(2); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); }).pipe(Effect.scoped), ); @@ -589,7 +663,7 @@ describe("DesktopBrowserHost", () => { expect(debuggee.sent).toEqual([1, 2, 3]); debuggee.release(); yield* settle; - expect(painting).toEqual([true, false, true, false]); + expect(painting).toEqual([true, false]); }).pipe(Effect.scoped), ); diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 613a4d686a5e9..9b8dd95c7fde8 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -119,6 +119,8 @@ export const make = Effect.gen(function* () { const nextScreenshot = (tab: AttachedTab) => { tab.screenshotTurn = null; tab.waitingScreenshots.shift()?.(); + releaseFocusWhenIdle(tab); + syncPainting(tab); }; /** @@ -149,7 +151,12 @@ export const make = Effect.gen(function* () { .then(resolve, reject); }; if (tab.screenshotTurn === null) start(); - else tab.waitingScreenshots.push(start); + else { + // The one ahead may be abandoned mid-capture; it needs frames to finish and free its turn. + tab.waitingScreenshots.push(start); + setFocusEmulation(tab, true); + syncPainting(tab); + } }); /** @@ -185,7 +192,13 @@ export const make = Effect.gen(function* () { } while (uses !== tab.focusUses); tab.focusReleasing = false; // The last pending command to settle starts another idle wait. - if (tab.focusPending.size > 0 || tab.screencasts.size > 0) return; + if ( + tab.focusPending.size > 0 || + tab.screencasts.size > 0 || + tab.waitingScreenshots.length > 0 + ) { + return; + } // The agent pauses to think between steps; its tab stays focused until the turn ends. if (tab.agentUsed && agentTurns.has(tab.key.threadId)) return; tab.agentUsed = false; @@ -202,7 +215,9 @@ export const make = Effect.gen(function* () { const syncPainting = (tab: AttachedTab) => { const active = tabs.get(keyOf(tab.key)) === tab && - (tab.screencasts.size > 0 || [...tab.focusPending.values()].some((hold) => hold.paints)); + (tab.screencasts.size > 0 || + tab.waitingScreenshots.length > 0 || + [...tab.focusPending.values()].some((hold) => hold.paints)); if (tab.painting === active) return; tab.painting = active; runFork(PubSub.publish(painting, { key: tab.key, active })); @@ -217,9 +232,8 @@ export const make = Effect.gen(function* () { ) => { const pending = {}; tab.focusPending.set(pending, { owner, paints: method === "Page.captureScreenshot" }); - // Viewers and recordings use extra sessions; only the main connection is the agent's, - // and Playwright's own housekeeping on it is not the agent using the tab. - if (owner === undefined && AGENT_INPUT_METHOD.test(method)) tab.agentUsed = true; + // Playwright's own housekeeping is not the agent using the tab. + if (AGENT_INPUT_METHOD.test(method)) tab.agentUsed = true; tab.focusUses += 1; setFocusEmulation(tab, true); releaseFocusWhenIdle(tab); @@ -250,7 +264,9 @@ export const make = Effect.gen(function* () { dropFocusHolds(tab); }; - const dropOwnerHolds = (tab: AttachedTab, owner: string | undefined) => { + /** A detached session, such as a closed viewer, waits on nothing it sent. */ + const dropSessionHolds = (tab: AttachedTab, owner: string) => { + tab.screencasts.delete(owner); for (const [pending, hold] of tab.focusPending) { if (hold.owner === owner) tab.focusPending.delete(pending); } @@ -258,22 +274,17 @@ export const make = Effect.gen(function* () { syncPainting(tab); }; - /** A detached session, such as a closed viewer, waits on nothing it sent. */ - const dropSessionHolds = (tab: AttachedTab, owner: string) => { - tab.screencasts.delete(owner); - dropOwnerHolds(tab, owner); - }; - /** - * A command the server stopped waiting on, such as a timed-out screenshot of a hidden - * guest, may never settle; the end of the turn that sent it ends its hold too. + * A command the server stopped waiting on, such as a timed-out screenshot or evaluation, + * may never settle; the turn's end ends every hold. The agent also sends on its own extra + * session, so this cannot spare other sessions; a viewer's screencast still holds. */ const endAgentTurn = (threadId: string) => { agentTurns.delete(threadId); for (const tab of tabs.values()) { if (tab.key.threadId !== threadId) continue; tab.agentUsed = false; - dropOwnerHolds(tab, undefined); + dropFocusHolds(tab); } }; @@ -288,8 +299,12 @@ export const make = Effect.gen(function* () { const relay: CdpRelayConnection = createCdpRelayConnection( { send: (method, params, sessionId, owner) => { - if (method === "Page.startScreencast") tab.screencasts.add(owner); - else if (method === "Page.stopScreencast") tab.screencasts.delete(owner); + // A child session's screencast is not the page's, such as a late one on a detached session. + if (sessionId === undefined && method === "Page.startScreencast") { + tab.screencasts.add(owner); + } else if (sessionId === undefined && method === "Page.stopScreencast") { + tab.screencasts.delete(owner); + } // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. const forward = (current?: () => boolean) => { const change = tab.focusChange; @@ -299,7 +314,6 @@ export const make = Effect.gen(function* () { : Promise.reject(new Error("The page this screenshot was for has moved on.")); return change === null ? run() : change.then(run); }; - // A screenshot holds focus only once its turn starts, not while queued behind another. return method === "Page.captureScreenshot" ? takeScreenshot(tab, relay, (current) => owner !== undefined && detachedOwners.has(owner) diff --git a/apps/server/src/preview/DesktopBrowserChannel.ts b/apps/server/src/preview/DesktopBrowserChannel.ts index a0ceaf6f369fa..d56b2a10a542b 100644 --- a/apps/server/src/preview/DesktopBrowserChannel.ts +++ b/apps/server/src/preview/DesktopBrowserChannel.ts @@ -49,7 +49,7 @@ export class DesktopBrowserChannel extends Context.Service< key: DesktopTabKey, pointer: { readonly phase: "move" | "click"; readonly x: number; readonly y: number }, ) => Effect.Effect; - /** Never fails: an undelivered change leaves the guest to its idle release. */ + /** Never fails. A failed write means a broken channel; the next backend starts with no turns. */ readonly turn: (threadId: string, active: boolean) => Effect.Effect; } >()("t3/preview/DesktopBrowserChannel") {} From 6600bc1cf5e0e3de70d7e35ca0875f2173f68a0f Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sun, 25 Oct 2026 18:34:57 -0700 Subject: [PATCH 12/17] fix(desktop): a queued screenshot paints only while its relay and session live Its liveness is also checked right before it sends, after any focus change, so a session that detaches meanwhile never gets an unheld capture. Co-Authored-By: Julius Marminge Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 61 ++++++++++++++++++- .../desktop/src/preview/DesktopBrowserHost.ts | 28 +++++---- 2 files changed, 76 insertions(+), 13 deletions(-) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index ae258ea0538e7..522419dee5431 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -497,6 +497,64 @@ describe("DesktopBrowserHost", () => { }).pipe(Effect.scoped), ); + it.effect("stops painting for a queued screenshot once its session detaches", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const painting = yield* watchPainting(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + // The agent's screenshot is abandoned at turn end; a viewer's waits behind it, then leaves. + yield* turn(true); + yield* command(host, 1, "Page.captureScreenshot"); + yield* settle; + yield* turn(false); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(2); + yield* raw({ id: 3, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" }); + yield* settle; + expect(painting).toEqual([true, false, true]); + yield* raw({ + id: 4, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + expect(yield* Queue.take(replies)).toBe(4); + expect(painting).toEqual([true, false, true, false]); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("never sends a screenshot whose session detached while focus was changing", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee({ deferFocus: true }); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" }); + yield* raw({ + id: 3, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + expect(yield* Queue.take(replies)).toBe(3); + debuggee.settleFocus(); + yield* settle; + expect(debuggee.sent).toEqual([]); + }).pipe(Effect.scoped), + ); + it.effect("ignores a screencast that arrives on a session after it detached", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; @@ -571,7 +629,8 @@ describe("DesktopBrowserHost", () => { yield* settle; debuggee.release(); expect(yield* Queue.take(replies)).toBe(4); - expect(yield* Queue.take(replies)).toBe(2); + // The dropped screenshot's error reply leaves as the first frees its turn. + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([3, 2]); yield* settle; expect(debuggee.sent).toEqual([2]); diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 9b8dd95c7fde8..7fd55d2cf21c1 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -44,7 +44,8 @@ interface AttachedTab { readonly debuggee: DesktopBrowserTabDebugger; relay: CdpRelayConnection | null; screenshotTurn: object | null; - readonly waitingScreenshots: Array<() => void>; + /** `live` while whoever queued it still waits on it. */ + readonly waitingScreenshots: Array<{ readonly start: () => void; readonly live: () => boolean }>; downloadDirectory: string | null; pendingDownloadGuid: string | null; agentInputAt: number; @@ -118,7 +119,7 @@ export const make = Effect.gen(function* () { const nextScreenshot = (tab: AttachedTab) => { tab.screenshotTurn = null; - tab.waitingScreenshots.shift()?.(); + tab.waitingScreenshots.shift()?.start(); releaseFocusWhenIdle(tab); syncPainting(tab); }; @@ -127,24 +128,26 @@ export const make = Effect.gen(function* () { * Chromium below 157 crashes the whole app when a guest's renderer exits with a * screenshot copy in flight (crbug.com/567321049). Overlapping screenshots make that * far likelier, and one the server gave up on stays pending in Chromium, so each guest - * gets one at a time. One queued by a relay released since then is dropped unsent. + * gets one at a time. One whose relay or session has since gone is dropped unsent. */ const takeScreenshot = ( tab: AttachedTab, relay: CdpRelayConnection, + ownerLive: () => boolean, /** Sends only while `current` holds, which focus emulation may outlast. */ capture: (current: () => boolean) => Promise, ) => new Promise((resolve, reject) => { + const live = () => tab.relay === relay && ownerLive(); const start = () => { - if (tab.relay !== relay) { + if (!live()) { reject(new Error("The preview connection that asked for this screenshot closed.")); nextScreenshot(tab); return; } const turn = {}; tab.screenshotTurn = turn; - capture(() => tab.screenshotTurn === turn && tab.relay === relay) + capture(() => tab.screenshotTurn === turn && live()) .finally(() => { if (tab.screenshotTurn === turn) nextScreenshot(tab); }) @@ -153,7 +156,7 @@ export const make = Effect.gen(function* () { if (tab.screenshotTurn === null) start(); else { // The one ahead may be abandoned mid-capture; it needs frames to finish and free its turn. - tab.waitingScreenshots.push(start); + tab.waitingScreenshots.push({ start, live }); setFocusEmulation(tab, true); syncPainting(tab); } @@ -195,7 +198,7 @@ export const make = Effect.gen(function* () { if ( tab.focusPending.size > 0 || tab.screencasts.size > 0 || - tab.waitingScreenshots.length > 0 + tab.waitingScreenshots.some((waiting) => waiting.live()) ) { return; } @@ -216,7 +219,7 @@ export const make = Effect.gen(function* () { const active = tabs.get(keyOf(tab.key)) === tab && (tab.screencasts.size > 0 || - tab.waitingScreenshots.length > 0 || + tab.waitingScreenshots.some((waiting) => waiting.live()) || [...tab.focusPending.values()].some((hold) => hold.paints)); if (tab.painting === active) return; tab.painting = active; @@ -315,10 +318,11 @@ export const make = Effect.gen(function* () { return change === null ? run() : change.then(run); }; return method === "Page.captureScreenshot" - ? takeScreenshot(tab, relay, (current) => - owner !== undefined && detachedOwners.has(owner) - ? Promise.reject(new Error("The session that asked for this screenshot closed.")) - : holdFocus(tab, method, owner, () => forward(current)), + ? takeScreenshot( + tab, + relay, + () => owner === undefined || !detachedOwners.has(owner), + (current) => holdFocus(tab, method, owner, () => forward(current)), ) : holdFocus(tab, method, owner, forward); }, From 23b707926f31df8bd7ad28c38d92683b3ec4dc65 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sun, 25 Oct 2026 18:36:24 -0700 Subject: [PATCH 13/17] fix(desktop): turn end releases queued screenshots, and a refused screencast holds nothing Co-Authored-By: Julius Marminge Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 52 ++++++++++++++++++- .../desktop/src/preview/DesktopBrowserHost.ts | 32 ++++++++---- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 522419dee5431..9a6ce1a5c0f97 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -31,7 +31,9 @@ const command = ( }), ); -const makeDebuggee = (options: { readonly deferFocus?: boolean } = {}) => { +const makeDebuggee = ( + options: { readonly deferFocus?: boolean; readonly refuse?: string } = {}, +) => { const emitter = new NodeEvents.EventEmitter(); const pending: Array<() => void> = []; const focusChanges: Array<() => void> = []; @@ -40,7 +42,11 @@ const makeDebuggee = (options: { readonly deferFocus?: boolean } = {}) => { const focusEmulation: Array = []; const debuggee = Object.assign(emitter, { sendCommand: (method: string, params?: { id?: number; enabled?: boolean }) => - new Promise((resolve) => { + new Promise((resolve, reject) => { + if (method === options.refuse) { + reject(new Error("Refused")); + return; + } if (method === "Target.getTargetInfo") { resolve({ targetInfo: { targetId: "GUEST" } }); return; @@ -555,6 +561,48 @@ describe("DesktopBrowserHost", () => { }).pipe(Effect.scoped), ); + it.effect("stops painting for a screenshot still queued when the agent's turn ends", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const painting = yield* watchPainting(host); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + yield* turn(true); + yield* command(host, 1, "Page.captureScreenshot"); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + yield* turn(false); + yield* settle; + expect(painting).toEqual([true, false]); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("lets a guest throttle when Chromium refuses a viewer's screencast", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee({ refuse: "Page.startScreencast" }); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const painting = yield* watchPainting(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + expect(yield* Queue.take(replies)).toBe(2); + yield* settle; + expect(painting).toEqual([true, false]); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + it.effect("ignores a screencast that arrives on a session after it detached", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 7fd55d2cf21c1..80cc990328a0a 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -44,8 +44,12 @@ interface AttachedTab { readonly debuggee: DesktopBrowserTabDebugger; relay: CdpRelayConnection | null; screenshotTurn: object | null; - /** `live` while whoever queued it still waits on it. */ - readonly waitingScreenshots: Array<{ readonly start: () => void; readonly live: () => boolean }>; + /** Each holds focus and painting while `held` and whoever queued it still waits on it. */ + readonly waitingScreenshots: Array<{ + readonly start: () => void; + readonly live: () => boolean; + held: boolean; + }>; downloadDirectory: string | null; pendingDownloadGuid: string | null; agentInputAt: number; @@ -156,7 +160,7 @@ export const make = Effect.gen(function* () { if (tab.screenshotTurn === null) start(); else { // The one ahead may be abandoned mid-capture; it needs frames to finish and free its turn. - tab.waitingScreenshots.push({ start, live }); + tab.waitingScreenshots.push({ start, live, held: true }); setFocusEmulation(tab, true); syncPainting(tab); } @@ -183,6 +187,9 @@ export const make = Effect.gen(function* () { tab.focusChange = change; }; + const waitingHolds = (tab: AttachedTab) => + tab.waitingScreenshots.some((waiting) => waiting.held && waiting.live()); + const releaseFocusWhenIdle = (tab: AttachedTab) => { if (tab.focusReleasing) return; tab.focusReleasing = true; @@ -195,11 +202,7 @@ export const make = Effect.gen(function* () { } while (uses !== tab.focusUses); tab.focusReleasing = false; // The last pending command to settle starts another idle wait. - if ( - tab.focusPending.size > 0 || - tab.screencasts.size > 0 || - tab.waitingScreenshots.some((waiting) => waiting.live()) - ) { + if (tab.focusPending.size > 0 || tab.screencasts.size > 0 || waitingHolds(tab)) { return; } // The agent pauses to think between steps; its tab stays focused until the turn ends. @@ -219,7 +222,7 @@ export const make = Effect.gen(function* () { const active = tabs.get(keyOf(tab.key)) === tab && (tab.screencasts.size > 0 || - tab.waitingScreenshots.some((waiting) => waiting.live()) || + waitingHolds(tab) || [...tab.focusPending.values()].some((hold) => hold.paints)); if (tab.painting === active) return; tab.painting = active; @@ -256,6 +259,7 @@ export const make = Effect.gen(function* () { */ const dropFocusHolds = (tab: AttachedTab) => { tab.focusPending.clear(); + for (const waiting of tab.waitingScreenshots) waiting.held = false; releaseFocusWhenIdle(tab); syncPainting(tab); }; @@ -324,7 +328,15 @@ export const make = Effect.gen(function* () { () => owner === undefined || !detachedOwners.has(owner), (current) => holdFocus(tab, method, owner, () => forward(current)), ) - : holdFocus(tab, method, owner, forward); + : method === "Page.startScreencast" + ? holdFocus(tab, method, owner, forward).catch((cause: unknown) => { + // A viewer can outlive a refused stream; nothing is streaming for it. + tab.screencasts.delete(owner); + releaseFocusWhenIdle(tab); + syncPainting(tab); + throw cause; + }) + : holdFocus(tab, method, owner, forward); }, sessionDetached: (owner) => { detachedOwners.add(owner); From ac85fbd63b403596e1ee78b6460054ad60294f01 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sun, 25 Oct 2026 18:37:35 -0700 Subject: [PATCH 14/17] fix(desktop): a person's input through a viewer is not agent use Co-Authored-By: Julius Marminge Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 29 +++++++++++++++++++ .../desktop/src/preview/DesktopBrowserHost.ts | 14 +++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 9a6ce1a5c0f97..72ca8edf3a0f1 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -603,6 +603,35 @@ describe("DesktopBrowserHost", () => { }).pipe(Effect.scoped), ); + it.effect("lets a guest throttle after a person's input through a viewer that then closed", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* host.handleCommandLine( + encodeJson({ type: "turn", threadId: key.threadId, active: true }), + ); + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ id: 3, method: "Input.dispatchMouseEvent", sessionId: "t3-preview-page-1" }); + yield* settle; + debuggee.release(); + yield* raw({ + id: 4, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + it.effect("ignores a screencast that arrives on a session after it detached", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 80cc990328a0a..e8862a23f2a92 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -22,7 +22,7 @@ const decodeCommand = Schema.decodeUnknownOption(Schema.fromJsonString(DesktopBr const lineEncoder = new TextEncoder(); const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; -/** The same commands as above, by method, for the main connection's sends. */ +/** The same commands as above, by method, for the relay's sends. */ const AGENT_INPUT_METHOD = /^(?:Input\.|Page\.navigate$|Runtime\.(?:evaluate|callFunctionOn)$)/; const AGENT_DOWNLOAD_WINDOW_MS = 5_000; const FOCUS_IDLE = "5 seconds"; @@ -238,8 +238,6 @@ export const make = Effect.gen(function* () { ) => { const pending = {}; tab.focusPending.set(pending, { owner, paints: method === "Page.captureScreenshot" }); - // Playwright's own housekeeping is not the agent using the tab. - if (AGENT_INPUT_METHOD.test(method)) tab.agentUsed = true; tab.focusUses += 1; setFocusEmulation(tab, true); releaseFocusWhenIdle(tab); @@ -303,15 +301,25 @@ export const make = Effect.gen(function* () { ? sendPageCommand(webContents, debuggee, method, params) : debuggee.sendCommand(method, params, sessionId); const detachedOwners = new Set(); + /** Viewers and recordings stream; the agent's own sessions never do. */ + const streamingOwners = new Set(); const relay: CdpRelayConnection = createCdpRelayConnection( { send: (method, params, sessionId, owner) => { // A child session's screencast is not the page's, such as a late one on a detached session. if (sessionId === undefined && method === "Page.startScreencast") { tab.screencasts.add(owner); + if (owner !== undefined) streamingOwners.add(owner); } else if (sessionId === undefined && method === "Page.stopScreencast") { tab.screencasts.delete(owner); } + // A person's input through a viewer, or Playwright's own housekeeping, is not the agent's. + if ( + AGENT_INPUT_METHOD.test(method) && + (owner === undefined || !streamingOwners.has(owner)) + ) { + tab.agentUsed = true; + } // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. const forward = (current?: () => boolean) => { const change = tab.focusChange; From 0ff70160399fd0454ee1495e48e3e1665d3ad771 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sun, 25 Oct 2026 18:48:53 -0700 Subject: [PATCH 15/17] fix(desktop): a screenshot queued past its turn's end goes unheld Co-Authored-By: Julius Marminge Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 6 +++ .../desktop/src/preview/DesktopBrowserHost.ts | 44 +++++++++++-------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 72ca8edf3a0f1..b0d1d36926b3c 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -577,6 +577,12 @@ describe("DesktopBrowserHost", () => { yield* turn(false); yield* settle; expect(painting).toEqual([true, false]); + + // The first answers late; the second still goes in turn, but holds nothing. + debuggee.releaseFirst(); + yield* settle; + expect(debuggee.sent).toEqual([1, 2]); + expect(painting).toEqual([true, false]); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); }).pipe(Effect.scoped), diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index e8862a23f2a92..ea832cfe6a058 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -138,29 +138,33 @@ export const make = Effect.gen(function* () { tab: AttachedTab, relay: CdpRelayConnection, ownerLive: () => boolean, - /** Sends only while `current` holds, which focus emulation may outlast. */ - capture: (current: () => boolean) => Promise, + /** Sends only while `current` holds; holds focus and painting only while `held`. */ + capture: (current: () => boolean, held: boolean) => Promise, ) => new Promise((resolve, reject) => { const live = () => tab.relay === relay && ownerLive(); - const start = () => { - if (!live()) { - reject(new Error("The preview connection that asked for this screenshot closed.")); - nextScreenshot(tab); - return; - } - const turn = {}; - tab.screenshotTurn = turn; - capture(() => tab.screenshotTurn === turn && live()) - .finally(() => { - if (tab.screenshotTurn === turn) nextScreenshot(tab); - }) - .then(resolve, reject); + const waiting = { + start: () => { + if (!live()) { + reject(new Error("The preview connection that asked for this screenshot closed.")); + nextScreenshot(tab); + return; + } + const turn = {}; + tab.screenshotTurn = turn; + capture(() => tab.screenshotTurn === turn && live(), waiting.held) + .finally(() => { + if (tab.screenshotTurn === turn) nextScreenshot(tab); + }) + .then(resolve, reject); + }, + live, + held: true, }; - if (tab.screenshotTurn === null) start(); + if (tab.screenshotTurn === null) waiting.start(); else { // The one ahead may be abandoned mid-capture; it needs frames to finish and free its turn. - tab.waitingScreenshots.push({ start, live, held: true }); + tab.waitingScreenshots.push(waiting); setFocusEmulation(tab, true); syncPainting(tab); } @@ -257,7 +261,6 @@ export const make = Effect.gen(function* () { */ const dropFocusHolds = (tab: AttachedTab) => { tab.focusPending.clear(); - for (const waiting of tab.waitingScreenshots) waiting.held = false; releaseFocusWhenIdle(tab); syncPainting(tab); }; @@ -289,6 +292,8 @@ export const make = Effect.gen(function* () { for (const tab of tabs.values()) { if (tab.key.threadId !== threadId) continue; tab.agentUsed = false; + // Still sent in turn, so the queue keeps moving, but no longer kept painting. + for (const waiting of tab.waitingScreenshots) waiting.held = false; dropFocusHolds(tab); } }; @@ -334,7 +339,8 @@ export const make = Effect.gen(function* () { tab, relay, () => owner === undefined || !detachedOwners.has(owner), - (current) => holdFocus(tab, method, owner, () => forward(current)), + (current, held) => + held ? holdFocus(tab, method, owner, () => forward(current)) : forward(current), ) : method === "Page.startScreencast" ? holdFocus(tab, method, owner, forward).catch((cause: unknown) => { From 05f0e36daa975862b2d80222f5dce506800ba147 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sun, 25 Oct 2026 18:59:23 -0700 Subject: [PATCH 16/17] fix(server): a child handed to another run still ends preview activity The settle check ran only for events this run accepts, so a handover that settled the run's agent turns left the thread active while background work kept the stream open. The relay test now asserts no send at all. Co-Authored-By: Julius Marminge Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/preview/CdpRelay.test.ts | 6 +- .../RunExecutionService.test.ts | 55 ++++++++++++++++++- .../orchestration-v2/RunExecutionService.ts | 5 ++ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/apps/desktop/src/preview/CdpRelay.test.ts b/apps/desktop/src/preview/CdpRelay.test.ts index e5acd3ece936d..1c1f9dc078ca0 100644 --- a/apps/desktop/src/preview/CdpRelay.test.ts +++ b/apps/desktop/src/preview/CdpRelay.test.ts @@ -115,11 +115,7 @@ describe("CDP relay", () => { ); await settle(); expect(written).toEqual([{ id: 1, result: {}, sessionId: "t3-preview-page" }]); - expect(send).not.toHaveBeenCalledWith( - "Emulation.setFocusEmulationEnabled", - { enabled: true }, - undefined, - ); + expect(send).not.toHaveBeenCalled(); }); it("routes tab events to the page session once the page is announced", async () => { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 86f68e593dc37..394cc1527cb7d 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -1859,6 +1859,46 @@ it.effect.each([null, "run:nested-sdk"] as const)( }), ); +// The handover event is not this run's, yet it is what settles the run's agent turns. +it.effect( + "ends preview activity when a transferred child settles a run kept open by background work", + () => + Effect.gen(function* () { + const active = yield* Deferred.make(); + yield* runBackgroundItemScenario( + "child-transfer-preview", + (ids) => { + const running = subagentEvent(ids, "running"); + if (running.type !== "subagent.updated") throw new Error("Missing subagent"); + const linked = { + ...running, + subagent: { + ...running.subagent, + origin: "provider_native" as const, + childThreadId: ids.childThreadId, + providerThreadId: null, + }, + }; + return [ + childThreadCreatedEvent(ids), + linked, + backgroundTurnItemEvent(ids, "command_execution", "running", 1), + rootTerminalEvent(ids, "completed"), + { ...linked, subagent: { ...linked.subagent, runId: RunId.make("run:replacement") } }, + ]; + }, + { + keepEventStreamOpen: true, + afterEvents: (activity) => + activity + .active(backgroundScenarioIds("child-transfer-preview").threadId) + .pipe(Effect.flatMap((value) => Deferred.succeed(active, value))), + }, + ).pipe(Effect.forkScoped); + assert.isFalse(yield* Deferred.await(active)); + }).pipe(Effect.scoped), +); + it.effect("keeps ingesting a failed root's background item when its work continues", () => Effect.gen(function* () { const observed = yield* runBackgroundItemScenario("bg-failed-continues", (ids) => [ @@ -3493,6 +3533,9 @@ function runBackgroundItemScenario( >; readonly onSubscribe?: Effect.Effect; readonly onEventsConsumed?: Effect.Effect; + readonly afterEvents?: ( + activity: PreviewAgentActivity.PreviewAgentActivity["Service"], + ) => Effect.Effect; readonly onFinalized?: ( events: ReadonlyArray, ) => Effect.Effect; @@ -3503,11 +3546,14 @@ function runBackgroundItemScenario( const providerInstanceId = ProviderInstanceId.make("codex"); const observed = yield* Ref.make>([]); const ingestionDone = yield* Deferred.make(); + const previewActivity = yield* PreviewAgentActivity.PreviewAgentActivity.pipe( + Effect.provide(PreviewAgentActivity.layer), + ); const layerTest = RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, - PreviewAgentActivity.layer, + Layer.succeed(PreviewAgentActivity.PreviewAgentActivity, previewActivity), Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({ @@ -3563,7 +3609,12 @@ function runBackgroundItemScenario( yield* options?.onSubscribe ?? Effect.void; const events = Stream.fromIterable(makeEvents(ids)).pipe( Stream.concat( - Stream.fromEffect(options?.onEventsConsumed ?? Effect.void).pipe(Stream.drain), + Stream.fromEffect( + Effect.andThen( + options?.onEventsConsumed ?? Effect.void, + options?.afterEvents?.(previewActivity) ?? Effect.void, + ), + ).pipe(Stream.drain), ), ); return { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 3d2c512a7970a..7878befe0e852 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1459,6 +1459,11 @@ export const layer: Layer.Layer< yield* finalizeRootRun(event); } yield* trackChildLifecycle(event, deliveredEvent !== null); + }), + ), + // Also after events this run does not accept, such as a child handed to another run. + Stream.tap(() => + Effect.gen(function* () { if (yield* agentTurnsSettled) yield* endPreviewActivity; }), ), From 184ffb42d28c23e24d3d12efdd892871c9992773 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sun, 25 Oct 2026 19:23:50 -0700 Subject: [PATCH 17/17] fix(desktop): a refused child screencast leaves the page's own running Co-Authored-By: Julius Marminge Co-Authored-By: Claude Opus 5.5 --- .../src/preview/DesktopBrowserHost.test.ts | 37 +++++++++++++++++-- .../desktop/src/preview/DesktopBrowserHost.ts | 2 +- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index b0d1d36926b3c..08fefd8d733f5 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -32,7 +32,10 @@ const command = ( ); const makeDebuggee = ( - options: { readonly deferFocus?: boolean; readonly refuse?: string } = {}, + options: { + readonly deferFocus?: boolean; + readonly refuse?: (method: string, sessionId: string | undefined) => boolean; + } = {}, ) => { const emitter = new NodeEvents.EventEmitter(); const pending: Array<() => void> = []; @@ -41,9 +44,13 @@ const makeDebuggee = ( const sendWaiters: Array<() => void> = []; const focusEmulation: Array = []; const debuggee = Object.assign(emitter, { - sendCommand: (method: string, params?: { id?: number; enabled?: boolean }) => + sendCommand: ( + method: string, + params?: { id?: number; enabled?: boolean }, + sessionId?: string, + ) => new Promise((resolve, reject) => { - if (method === options.refuse) { + if (options.refuse?.(method, sessionId)) { reject(new Error("Refused")); return; } @@ -591,7 +598,7 @@ describe("DesktopBrowserHost", () => { it.effect("lets a guest throttle when Chromium refuses a viewer's screencast", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; - const debuggee = makeDebuggee({ refuse: "Page.startScreencast" }); + const debuggee = makeDebuggee({ refuse: (method) => method === "Page.startScreencast" }); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); const painting = yield* watchPainting(host); @@ -638,6 +645,28 @@ describe("DesktopBrowserHost", () => { }).pipe(Effect.scoped), ); + it.effect("keeps a page's screencast when a child session's screencast is refused", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee({ + refuse: (method, sessionId) => method === "Page.startScreencast" && sessionId !== undefined, + }); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* command(host, 1, "Page.startScreencast"); + yield* settle; + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "child-frame" }); + expect(yield* Queue.take(replies)).toBe(2); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + }).pipe(Effect.scoped), + ); + it.effect("ignores a screencast that arrives on a session after it detached", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index ea832cfe6a058..30f252e3a677e 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -345,7 +345,7 @@ export const make = Effect.gen(function* () { : method === "Page.startScreencast" ? holdFocus(tab, method, owner, forward).catch((cause: unknown) => { // A viewer can outlive a refused stream; nothing is streaming for it. - tab.screencasts.delete(owner); + if (sessionId === undefined && tab.relay === relay) tab.screencasts.delete(owner); releaseFocusWhenIdle(tab); syncPainting(tab); throw cause;