diff --git a/apps/desktop/src/preview/CdpRelay.test.ts b/apps/desktop/src/preview/CdpRelay.test.ts index ec314d1f9d478..1c1f9dc078ca0 100644 --- a/apps/desktop/src/preview/CdpRelay.test.ts +++ b/apps/desktop/src/preview/CdpRelay.test.ts @@ -65,8 +65,8 @@ describe("CDP relay", () => { ]); pending[0]!(); await settle(); - expect(send).toHaveBeenNthCalledWith(1, "Page.captureScreenshot", {}, undefined); - expect(send).toHaveBeenNthCalledWith(2, "DOM.enable", {}, "iframe-session"); + expect(send).toHaveBeenNthCalledWith(1, "Page.captureScreenshot", {}, undefined, undefined); + expect(send).toHaveBeenNthCalledWith(2, "DOM.enable", {}, "iframe-session", undefined); expect(written.at(-1)).toEqual({ id: 5, result: { method: "Page.captureScreenshot" }, @@ -92,7 +92,30 @@ describe("CDP relay", () => { }), ); await settle(); - expect(send.mock.calls).toEqual([["Runtime.runIfWaitingForDebugger", {}, "iframe-session"]]); + expect(send.mock.calls).toEqual([ + ["Runtime.runIfWaitingForDebugger", {}, "iframe-session", undefined], + ]); + }); + + // Playwright enables focus emulation on every page it connects to, and Chromium then + // counts the guest as captured: a preview the panel hides keeps its timers unthrottled. + it("acknowledges Playwright's focus emulation without forcing it onto the guest", async () => { + const send = vi.fn(async () => ({})); + const written: Array> = []; + const relay = createCdpRelayConnection(makeTarget(send), (raw) => + written.push(JSON.parse(raw)), + ); + relay.receive( + JSON.stringify({ + id: 1, + method: "Emulation.setFocusEmulationEnabled", + params: { enabled: true }, + sessionId: "t3-preview-page", + }), + ); + await settle(); + expect(written).toEqual([{ id: 1, result: {}, sessionId: "t3-preview-page" }]); + expect(send).not.toHaveBeenCalled(); }); it("routes tab events to the page session once the page is announced", async () => { @@ -147,7 +170,7 @@ describe("CDP relay extra sessions", () => { expect(written.find((m) => m["id"] === 4)).toHaveProperty("error"); relay.receive(JSON.stringify({ id: 5, method: "Page.startScreencast", sessionId: extra })); await settle(); - expect(send).toHaveBeenLastCalledWith("Page.startScreencast", {}, undefined); + expect(send).toHaveBeenLastCalledWith("Page.startScreencast", {}, undefined, extra); written.length = 0; relay.event("Page.screencastFrame", { data: "x" }, undefined); expect(written.map((m) => m["sessionId"])).toEqual(["t3-preview-page", extra]); diff --git a/apps/desktop/src/preview/CdpRelay.ts b/apps/desktop/src/preview/CdpRelay.ts index 82264cc4b870b..948bc361f5d92 100644 --- a/apps/desktop/src/preview/CdpRelay.ts +++ b/apps/desktop/src/preview/CdpRelay.ts @@ -5,7 +5,10 @@ export interface CdpRelayTarget { method: string, params: Record, sessionId: string | undefined, + /** The extra session that sent it, such as a viewer's; the main connection has none. */ + owner?: string, ) => Promise; + readonly sessionDetached?: (owner: string) => void; /** The page's own CDP target id; Playwright keys the main frame by it. */ readonly targetId: () => Promise; readonly url: () => string; @@ -111,8 +114,10 @@ export function createCdpRelayConnection( } case "Target.detachFromTarget": { const sessionId = command.params?.["sessionId"]; - if (typeof sessionId === "string" && sessionId !== PAGE_SESSION_ID) + if (typeof sessionId === "string" && sessionId !== PAGE_SESSION_ID) { sessions.delete(sessionId); + target.sessionDetached?.(sessionId); + } return {}; } case "Target.setAutoAttach": { @@ -133,11 +138,15 @@ export function createCdpRelayConnection( const pageCommand = (command: CdpCommand): Promise => { const child = sessions.get(command.sessionId!) === "page" ? undefined : command.sessionId; + const owner = + child === undefined && command.sessionId !== PAGE_SESSION_ID ? command.sessionId : undefined; // Child frames/workers pause on auto-attach until resumed; only the main page can skip this command. if (command.method === "Runtime.runIfWaitingForDebugger" && child === undefined) { return Promise.resolve({}); } - return target.send(command.method, command.params ?? {}, child); + // Playwright asks for this on every page; the host emulates focus only while it is in use. + if (command.method === "Emulation.setFocusEmulationEnabled") return Promise.resolve({}); + return target.send(command.method, command.params ?? {}, child, owner); }; return { diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 50039c4e4b04e..08fefd8d733f5 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -6,6 +6,7 @@ import * as Fiber from "effect/Fiber"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import * as NodeEvents from "node:events"; import * as DesktopBrowserHost from "./DesktopBrowserHost.ts"; @@ -30,18 +31,39 @@ const command = ( }), ); -const makeDebuggee = () => { +const makeDebuggee = ( + options: { + readonly deferFocus?: boolean; + readonly refuse?: (method: string, sessionId: string | undefined) => boolean; + } = {}, +) => { const emitter = new NodeEvents.EventEmitter(); const pending: Array<() => void> = []; + const focusChanges: Array<() => void> = []; const sent: Array = []; const sendWaiters: Array<() => void> = []; + const focusEmulation: Array = []; const debuggee = Object.assign(emitter, { - sendCommand: (method: string, params?: { id?: number }) => - new Promise((resolve) => { + sendCommand: ( + method: string, + params?: { id?: number; enabled?: boolean }, + sessionId?: string, + ) => + new Promise((resolve, reject) => { + if (options.refuse?.(method, sessionId)) { + reject(new Error("Refused")); + return; + } if (method === "Target.getTargetInfo") { resolve({ targetInfo: { targetId: "GUEST" } }); return; } + if (method === "Emulation.setFocusEmulationEnabled") { + focusEmulation.push(params?.enabled === true); + if (options.deferFocus) focusChanges.push(() => resolve({})); + else resolve({}); + return; + } sent.push(params?.id); pending.push(() => resolve({ method })); sendWaiters.shift()?.(); @@ -61,12 +83,18 @@ const makeDebuggee = () => { navigate: () => webContents.emit("did-navigate"), /** Ids of the commands the debugger received, in order. */ sent, + focusEmulation, nextSend: () => new Promise((resolve) => sendWaiters.push(resolve)), release: () => pending.splice(0).forEach((resolve) => resolve()), releaseFirst: () => pending.shift()?.(), + settleFocus: () => focusChanges.splice(0).forEach((resolve) => resolve()), }; }; +/** Commands wait behind a focus emulation change; it settles within a macrotask. */ +// oxlint-disable-next-line t3code/no-real-time-wait-in-tests -- a zero-delay macrotask boundary, not a wait for time +const settle = Effect.promise(() => new Promise((resolve) => setImmediate(resolve))); + const takeEvents = (host: DesktopBrowserHost.DesktopBrowserHost["Service"], count: number) => host.events.pipe( Stream.take(count), @@ -93,6 +121,18 @@ const watchReplies = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) => return replies; }); +/** Whether the client should paint the guest, each time that changes. */ +const watchPainting = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) => + Effect.gen(function* () { + const painting: Array = []; + yield* host.painting.pipe( + Stream.runForEach(({ active }) => Effect.sync(() => painting.push(active))), + Effect.forkScoped, + ); + yield* Effect.yieldNow; + return painting; + }); + describe("DesktopBrowserHost", () => { it.effect("announces tabs already attached to a backend that starts later", () => Effect.gen(function* () { @@ -123,6 +163,7 @@ describe("DesktopBrowserHost", () => { yield* command(1, "Page.captureScreenshot"); yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); yield* command(2, "DOM.enable"); + yield* settle; debuggee.release(); const [, reply] = yield* Fiber.join(reader); @@ -141,6 +182,7 @@ describe("DesktopBrowserHost", () => { yield* command(host, 1, "Page.captureScreenshot"); yield* command(host, 2, "Page.captureScreenshot"); yield* command(host, 3, "DOM.enable"); + yield* settle; expect(debuggee.sent).toEqual([1, 3]); debuggee.releaseFirst(); @@ -159,6 +201,7 @@ describe("DesktopBrowserHost", () => { yield* command(host, 1, "Page.captureScreenshot"); yield* command(host, 2, "Page.captureScreenshot"); yield* command(host, 3, "Page.captureScreenshot"); + yield* settle; debuggee.navigate(); expect(debuggee.sent).toEqual([1, 2]); @@ -181,6 +224,7 @@ describe("DesktopBrowserHost", () => { yield* command(host, 1, "Page.captureScreenshot"); yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); yield* command(host, 3, "Page.captureScreenshot"); expect(debuggee.sent).toEqual([1]); @@ -192,6 +236,682 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect("emulates focus while the server drives a guest, then lets a hidden one throttle", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + + yield* command(host, 1, "DOM.enable"); + yield* settle; + expect(debuggee.focusEmulation).toEqual([true]); + expect(debuggee.sent).toEqual([1]); + debuggee.release(); + yield* settle; + + yield* TestClock.adjust("4 seconds"); + yield* command(host, 2, "Input.dispatchMouseEvent"); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("4 seconds"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* TestClock.adjust("10 seconds"); + expect(debuggee.focusEmulation).toEqual([true, false]); + + yield* command(host, 3, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.focusEmulation).toEqual([true, false, true]); + expect(debuggee.sent).toEqual([1, 2, 3]); + }), + ); + + it.effect("keeps a guest focused while a command it is running is still pending", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + + // An evaluation awaiting the page's next frame, which a throttled guest never paints. + yield* command(host, 1, "Runtime.evaluate"); + yield* settle; + expect(debuggee.sent).toEqual([1]); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(1); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("lets a guest throttle once Chromium can no longer answer its pending command", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + + yield* command(host, 1, "Runtime.evaluate"); + yield* settle; + yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + + // A command the old document never answers. + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.sent).toEqual([1, 2]); + debuggee.navigate(); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false, true, false]); + }), + ); + + it.effect("keeps a guest focused for the screenshot a navigation lets start", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + + yield* command(host, 1, "Page.captureScreenshot"); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + debuggee.navigate(); + expect(debuggee.sent).toEqual([1, 2]); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + }), + ); + + it.effect("never sends a screenshot a navigation superseded while focus was changing", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee({ deferFocus: true }); + host.attach(key, debuggee.tab); + + yield* command(host, 1, "Page.captureScreenshot"); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + debuggee.navigate(); + debuggee.settleFocus(); + yield* settle; + // Two captures in flight would undo the one-at-a-time guard. + expect(debuggee.sent).toEqual([2]); + }), + ); + + it.effect("keeps a guest focused when its turn starts after the agent's first command", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + yield* command(host, 1, "Input.dispatchKeyEvent"); + yield* settle; + yield* turn(true); + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* turn(false); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }), + ); + + it.effect("lets a guest throttle through a turn that only sent it housekeeping", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + yield* host.handleCommandLine( + encodeJson({ type: "turn", threadId: key.threadId, active: true }), + ); + + // Playwright re-enabling a domain after a reload is not the agent using the tab. + yield* command(host, 1, "Runtime.enable"); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + + yield* command(host, 2, "Runtime.evaluate"); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false, true]); + }), + ); + + it.effect("keeps a guest the agent used focused through its turn's pauses", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + yield* turn(true); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([]); + + yield* command(host, 1, "Input.dispatchKeyEvent"); + yield* settle; + debuggee.release(); + yield* settle; + // The model thinks between steps; a focus-sensitive menu must not blur meanwhile. + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* turn(false); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }), + ); + + it.effect( + "ends a command's hold with the turn that sent it, even if Chromium never answers", + () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + yield* host.handleCommandLine( + encodeJson({ type: "turn", threadId: key.threadId, active: true }), + ); + + // A screenshot of a hidden guest the server already gave up on. + yield* command(host, 1, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.sent).toEqual([1]); + yield* host.handleCommandLine( + encodeJson({ type: "turn", threadId: key.threadId, active: false }), + ); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }), + ); + + it.effect("ends the agent's own session's holds with its turn, but not a viewer's stream", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + // The server's own CDP session, evaluating a promise that never settles. + yield* turn(true); + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Runtime.evaluate", sessionId: "t3-preview-page-1" }); + yield* settle; + yield* turn(false); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + + // A viewer streaming the guest keeps it focused through a turn's end. + yield* turn(true); + yield* raw({ id: 3, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(3); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" }); + yield* settle; + yield* turn(false); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false, true]); + }).pipe(Effect.scoped), + ); + + it.effect("paints for a screenshot queued behind one its viewer abandoned mid-capture", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const painting = yield* watchPainting(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" }); + yield* settle; + yield* raw({ + id: 3, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + expect(yield* Queue.take(replies)).toBe(3); + expect(painting).toEqual([true, false]); + + // The agent's screenshot waits its turn; painting lets the abandoned one finish first. + yield* command(host, 4, "Page.captureScreenshot"); + yield* settle; + expect(painting).toEqual([true, false, true]); + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(2); + yield* settle; + expect(debuggee.sent).toEqual([undefined, 4]); + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(4); + yield* settle; + expect(painting).toEqual([true, false, true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("stops painting for a queued screenshot once its session detaches", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const painting = yield* watchPainting(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + // The agent's screenshot is abandoned at turn end; a viewer's waits behind it, then leaves. + yield* turn(true); + yield* command(host, 1, "Page.captureScreenshot"); + yield* settle; + yield* turn(false); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(2); + yield* raw({ id: 3, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" }); + yield* settle; + expect(painting).toEqual([true, false, true]); + yield* raw({ + id: 4, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + expect(yield* Queue.take(replies)).toBe(4); + expect(painting).toEqual([true, false, true, false]); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("never sends a screenshot whose session detached while focus was changing", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee({ deferFocus: true }); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" }); + yield* raw({ + id: 3, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + expect(yield* Queue.take(replies)).toBe(3); + debuggee.settleFocus(); + yield* settle; + expect(debuggee.sent).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect("stops painting for a screenshot still queued when the agent's turn ends", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const painting = yield* watchPainting(host); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + yield* turn(true); + yield* command(host, 1, "Page.captureScreenshot"); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + yield* turn(false); + yield* settle; + expect(painting).toEqual([true, false]); + + // The first answers late; the second still goes in turn, but holds nothing. + debuggee.releaseFirst(); + yield* settle; + expect(debuggee.sent).toEqual([1, 2]); + expect(painting).toEqual([true, false]); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("lets a guest throttle when Chromium refuses a viewer's screencast", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee({ refuse: (method) => method === "Page.startScreencast" }); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const painting = yield* watchPainting(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + expect(yield* Queue.take(replies)).toBe(2); + yield* settle; + expect(painting).toEqual([true, false]); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("lets a guest throttle after a person's input through a viewer that then closed", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* host.handleCommandLine( + encodeJson({ type: "turn", threadId: key.threadId, active: true }), + ); + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ id: 3, method: "Input.dispatchMouseEvent", sessionId: "t3-preview-page-1" }); + yield* settle; + debuggee.release(); + yield* raw({ + id: 4, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("keeps a page's screencast when a child session's screencast is refused", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee({ + refuse: (method, sessionId) => method === "Page.startScreencast" && sessionId !== undefined, + }); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* command(host, 1, "Page.startScreencast"); + yield* settle; + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "child-frame" }); + expect(yield* Queue.take(replies)).toBe(2); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + }).pipe(Effect.scoped), + ); + + it.effect("ignores a screencast that arrives on a session after it detached", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ + id: 2, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + expect(yield* Queue.take(replies)).toBe(2); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("lets a guest throttle once the session waiting on its command detaches", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + // A viewer's own CDP session, closed while its still of a hidden guest is pending. + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ id: 2, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" }); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* raw({ + id: 3, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("never sends a screenshot queued by a session that has since detached", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* command(host, 2, "Page.captureScreenshot"); + yield* raw({ id: 3, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" }); + yield* raw({ + id: 4, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + yield* settle; + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(4); + // The dropped screenshot's error reply leaves as the first frees its turn. + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([3, 2]); + yield* settle; + expect(debuggee.sent).toEqual([2]); + + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("keeps a streamed guest focused until its screencast ends", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + + yield* command(host, 1, "Page.startScreencast"); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* command(host, 2, "Page.stopScreencast"); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + + // A viewer whose connection went away without stopping its screencast. + yield* command(host, 3, "Page.startScreencast"); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false, true, false]); + }), + ); + + it.effect("keeps a guest focused while any of its screencasts runs", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + // Two viewers, or a viewer and a recording, each on its own session. + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([1, 2]); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" }); + yield* raw({ id: 5, method: "Page.stopScreencast", sessionId: "t3-preview-page-1" }); + yield* settle; + debuggee.release(); + yield* settle; + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true]); + + yield* raw({ + id: 6, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-2" }, + }); + yield* TestClock.adjust("1 minute"); + expect(debuggee.focusEmulation).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("paints a hidden guest only while the screenshot it is taking is pending", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const painting = yield* watchPainting(host); + + yield* command(host, 1, "DOM.enable"); + yield* settle; + expect(painting).toEqual([]); + + yield* command(host, 2, "Page.captureScreenshot"); + yield* command(host, 3, "Page.captureScreenshot"); + yield* settle; + expect(painting).toEqual([true]); + + // The first answers once the guest paints; the queued one then takes its turn. + debuggee.release(); + yield* settle; + expect(debuggee.sent).toEqual([1, 2, 3]); + debuggee.release(); + yield* settle; + expect(painting).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("stops painting for a screenshot Chromium never answers", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const painting = yield* watchPainting(host); + const turn = (active: boolean) => + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active })); + + // The server gave up on it, and the agent's turn ends. + yield* turn(true); + yield* command(host, 1, "Page.captureScreenshot"); + yield* settle; + yield* turn(false); + yield* settle; + expect(painting).toEqual([true, false]); + + // The page loads a new document, which also frees the screenshot turn. + debuggee.navigate(); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + debuggee.navigate(); + yield* settle; + expect(painting).toEqual([true, false, true, false]); + + // The server releases the relay. + yield* command(host, 3, "Page.captureScreenshot"); + yield* settle; + yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); + yield* settle; + expect(painting).toEqual([true, false, true, false, true, false]); + + // The guest detaches; a replacement starts with no screenshot pending. + const replacement = makeDebuggee(); + host.attach(key, replacement.tab); + yield* command(host, 4, "Page.captureScreenshot"); + yield* settle; + host.detach(key); + yield* settle; + expect(painting).toEqual([true, false, true, false, true, false, true, false]); + }).pipe(Effect.scoped), + ); + + it.effect("paints a streamed guest until its last screencast ends", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const painting = yield* watchPainting(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([1, 2]); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" }); + yield* raw({ id: 5, method: "Page.stopScreencast", sessionId: "t3-preview-page-1" }); + yield* settle; + debuggee.release(); + yield* settle; + expect(painting).toEqual([true]); + + yield* raw({ + id: 6, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-2" }, + }); + yield* settle; + expect(painting).toEqual([true, false]); + }).pipe(Effect.scoped), + ); + it.effect("saves a server tab's download under its CDP guid where the server asked", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 849ae658a8219..30f252e3a677e 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -22,7 +22,10 @@ const decodeCommand = Schema.decodeUnknownOption(Schema.fromJsonString(DesktopBr const lineEncoder = new TextEncoder(); const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; +/** The same commands as above, by method, for the relay's sends. */ +const AGENT_INPUT_METHOD = /^(?:Input\.|Page\.navigate$|Runtime\.(?:evaluate|callFunctionOn)$)/; const AGENT_DOWNLOAD_WINDOW_MS = 5_000; +const FOCUS_IDLE = "5 seconds"; export interface DesktopBrowserTabKey { readonly threadId: string; @@ -41,10 +44,33 @@ interface AttachedTab { readonly debuggee: DesktopBrowserTabDebugger; relay: CdpRelayConnection | null; screenshotTurn: object | null; - readonly waitingScreenshots: Array<() => void>; + /** Each holds focus and painting while `held` and whoever queued it still waits on it. */ + readonly waitingScreenshots: Array<{ + readonly start: () => void; + readonly live: () => boolean; + held: boolean; + }>; downloadDirectory: string | null; pendingDownloadGuid: string | null; agentInputAt: number; + focusEmulated: boolean; + focusChange: Promise | null; + focusUses: number; + /** Each pending command, by the extra session waiting on it; the main connection's have none. */ + readonly focusPending: Map< + object, + { readonly owner: string | undefined; readonly paints: boolean } + >; + /** Whether the client keeps the guest painting, as published on `painting`. */ + painting: boolean; + /** + * Driven by the agent since its thread's last turn ended or the guest last throttled. A + * turn's start can reach us after its first command, so this holds focus once it does. + */ + agentUsed: boolean; + focusReleasing: boolean; + /** The sessions streaming the guest; one stopping leaves the others' screencasts running. */ + readonly screencasts: Set; readonly onMessage: ( event: Electron.Event, method: string, @@ -69,6 +95,11 @@ export class DesktopBrowserHost extends Context.Service< readonly x: number; readonly y: number; }>; + /** Whether a guest needs its frames painted while the panel hides it. */ + readonly painting: Stream.Stream<{ + readonly key: DesktopBrowserTabKey; + readonly active: boolean; + }>; } >()("@t3tools/desktop/preview/DesktopBrowserHost") {} @@ -80,44 +111,192 @@ export const make = Effect.gen(function* () { readonly x: number; readonly y: number; }>(16); + const painting = yield* PubSub.unbounded<{ + readonly key: DesktopBrowserTabKey; + readonly active: boolean; + }>(); const runFork = Effect.runForkWith(yield* Effect.context()); const tabs = new Map(); + /** Threads whose agent turn runs, by the server's account. */ + const agentTurns = new Set(); const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event)); const nextScreenshot = (tab: AttachedTab) => { tab.screenshotTurn = null; - tab.waitingScreenshots.shift()?.(); + tab.waitingScreenshots.shift()?.start(); + releaseFocusWhenIdle(tab); + syncPainting(tab); }; /** * Chromium below 157 crashes the whole app when a guest's renderer exits with a * screenshot copy in flight (crbug.com/567321049). Overlapping screenshots make that * far likelier, and one the server gave up on stays pending in Chromium, so each guest - * gets one at a time. One queued by a relay released since then is dropped unsent. + * gets one at a time. One whose relay or session has since gone is dropped unsent. */ const takeScreenshot = ( tab: AttachedTab, relay: CdpRelayConnection, - capture: () => Promise, + ownerLive: () => boolean, + /** Sends only while `current` holds; holds focus and painting only while `held`. */ + capture: (current: () => boolean, held: boolean) => Promise, ) => new Promise((resolve, reject) => { - const start = () => { - if (tab.relay !== relay) { - reject(new Error("The preview connection that asked for this screenshot closed.")); - nextScreenshot(tab); + const live = () => tab.relay === relay && ownerLive(); + const waiting = { + start: () => { + if (!live()) { + reject(new Error("The preview connection that asked for this screenshot closed.")); + nextScreenshot(tab); + return; + } + const turn = {}; + tab.screenshotTurn = turn; + capture(() => tab.screenshotTurn === turn && live(), waiting.held) + .finally(() => { + if (tab.screenshotTurn === turn) nextScreenshot(tab); + }) + .then(resolve, reject); + }, + live, + held: true, + }; + if (tab.screenshotTurn === null) waiting.start(); + else { + // The one ahead may be abandoned mid-capture; it needs frames to finish and free its turn. + tab.waitingScreenshots.push(waiting); + setFocusEmulation(tab, true); + syncPainting(tab); + } + }); + + /** + * Chromium counts a focus-emulated guest as captured, so a guest the panel hides would + * never throttle. Without it, a hidden guest paints no frames and focus() fires no focus + * events, so the guest is emulated focused while the server drives it or streams it. + */ + const setFocusEmulation = (tab: AttachedTab, enabled: boolean) => { + if (tab.focusEmulated === enabled) return; + tab.focusEmulated = enabled; + // A guest that refuses it, or a debugger already gone, behaves as unfocused. + const change: Promise = tab.debuggee.debugger + .sendCommand("Emulation.setFocusEmulationEnabled", { enabled }) + .then( + () => undefined, + () => undefined, + ) + .finally(() => { + if (tab.focusChange === change) tab.focusChange = null; + }); + tab.focusChange = change; + }; + + const waitingHolds = (tab: AttachedTab) => + tab.waitingScreenshots.some((waiting) => waiting.held && waiting.live()); + + const releaseFocusWhenIdle = (tab: AttachedTab) => { + if (tab.focusReleasing) return; + tab.focusReleasing = true; + runFork( + Effect.gen(function* () { + let uses: number; + do { + uses = tab.focusUses; + yield* Effect.sleep(FOCUS_IDLE); + } while (uses !== tab.focusUses); + tab.focusReleasing = false; + // The last pending command to settle starts another idle wait. + if (tab.focusPending.size > 0 || tab.screencasts.size > 0 || waitingHolds(tab)) { return; } - const turn = {}; - tab.screenshotTurn = turn; - capture() - .finally(() => { - if (tab.screenshotTurn === turn) nextScreenshot(tab); - }) - .then(resolve, reject); - }; - if (tab.screenshotTurn === null) start(); - else tab.waitingScreenshots.push(start); + // The agent pauses to think between steps; its tab stays focused until the turn ends. + if (tab.agentUsed && agentTurns.has(tab.key.threadId)) return; + tab.agentUsed = false; + if (tabs.get(keyOf(tab.key)) === tab) setFocusEmulation(tab, false); + }), + ); + }; + + /** + * Electron composites no offscreen guest, so its screenshot or screencast waits for a frame + * that never comes. The client paints the guest behind the app while one is pending; the + * capture then finishes on its own, and the guest goes back offscreen once none is. + */ + const syncPainting = (tab: AttachedTab) => { + const active = + tabs.get(keyOf(tab.key)) === tab && + (tab.screencasts.size > 0 || + waitingHolds(tab) || + [...tab.focusPending.values()].some((hold) => hold.paints)); + if (tab.painting === active) return; + tab.painting = active; + runFork(PubSub.publish(painting, { key: tab.key, active })); + }; + + /** A command may wait on the page's next frame, so the guest stays focused until it settles. */ + const holdFocus = ( + tab: AttachedTab, + method: string, + owner: string | undefined, + work: () => Promise, + ) => { + const pending = {}; + tab.focusPending.set(pending, { owner, paints: method === "Page.captureScreenshot" }); + tab.focusUses += 1; + setFocusEmulation(tab, true); + releaseFocusWhenIdle(tab); + syncPainting(tab); + return work().finally(() => { + if (!tab.focusPending.delete(pending)) return; + tab.focusUses += 1; + releaseFocusWhenIdle(tab); + syncPainting(tab); }); + }; + + /** + * Chromium can leave a command unanswered, so the screenshot guard's release points + * also end every hold: a released relay has no one waiting, and a new document answers + * nothing pending from the old one. A released relay's screencast has no viewer to stop it. + */ + const dropFocusHolds = (tab: AttachedTab) => { + tab.focusPending.clear(); + releaseFocusWhenIdle(tab); + syncPainting(tab); + }; + + const dropRelay = (tab: AttachedTab) => { + tab.relay = null; + tab.screencasts.clear(); + tab.agentUsed = false; + dropFocusHolds(tab); + }; + + /** A detached session, such as a closed viewer, waits on nothing it sent. */ + const dropSessionHolds = (tab: AttachedTab, owner: string) => { + tab.screencasts.delete(owner); + for (const [pending, hold] of tab.focusPending) { + if (hold.owner === owner) tab.focusPending.delete(pending); + } + releaseFocusWhenIdle(tab); + syncPainting(tab); + }; + + /** + * A command the server stopped waiting on, such as a timed-out screenshot or evaluation, + * may never settle; the turn's end ends every hold. The agent also sends on its own extra + * session, so this cannot spare other sessions; a viewer's screencast still holds. + */ + const endAgentTurn = (threadId: string) => { + agentTurns.delete(threadId); + for (const tab of tabs.values()) { + if (tab.key.threadId !== threadId) continue; + tab.agentUsed = false; + // Still sent in turn, so the queue keeps moving, but no longer kept painting. + for (const waiting of tab.waitingScreenshots) waiting.held = false; + dropFocusHolds(tab); + } + }; const relayFor = (tab: AttachedTab) => { if (tab.relay) return tab.relay; @@ -126,12 +305,57 @@ export const make = Effect.gen(function* () { sessionId === undefined ? sendPageCommand(webContents, debuggee, method, params) : debuggee.sendCommand(method, params, sessionId); + const detachedOwners = new Set(); + /** Viewers and recordings stream; the agent's own sessions never do. */ + const streamingOwners = new Set(); const relay: CdpRelayConnection = createCdpRelayConnection( { - send: (method, params, sessionId) => - method === "Page.captureScreenshot" - ? takeScreenshot(tab, relay, () => send(method, params, sessionId)) - : send(method, params, sessionId), + send: (method, params, sessionId, owner) => { + // A child session's screencast is not the page's, such as a late one on a detached session. + if (sessionId === undefined && method === "Page.startScreencast") { + tab.screencasts.add(owner); + if (owner !== undefined) streamingOwners.add(owner); + } else if (sessionId === undefined && method === "Page.stopScreencast") { + tab.screencasts.delete(owner); + } + // A person's input through a viewer, or Playwright's own housekeeping, is not the agent's. + if ( + AGENT_INPUT_METHOD.test(method) && + (owner === undefined || !streamingOwners.has(owner)) + ) { + tab.agentUsed = true; + } + // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. + const forward = (current?: () => boolean) => { + const change = tab.focusChange; + const run = () => + current === undefined || current() + ? send(method, params, sessionId) + : Promise.reject(new Error("The page this screenshot was for has moved on.")); + return change === null ? run() : change.then(run); + }; + return method === "Page.captureScreenshot" + ? takeScreenshot( + tab, + relay, + () => owner === undefined || !detachedOwners.has(owner), + (current, held) => + held ? holdFocus(tab, method, owner, () => forward(current)) : forward(current), + ) + : method === "Page.startScreencast" + ? holdFocus(tab, method, owner, forward).catch((cause: unknown) => { + // A viewer can outlive a refused stream; nothing is streaming for it. + if (sessionId === undefined && tab.relay === relay) tab.screencasts.delete(owner); + releaseFocusWhenIdle(tab); + syncPainting(tab); + throw cause; + }) + : holdFocus(tab, method, owner, forward); + }, + sessionDetached: (owner) => { + detachedOwners.add(owner); + dropSessionHolds(tab, owner); + }, targetId: () => debuggee .sendCommand("Target.getTargetInfo") @@ -172,6 +396,7 @@ export const make = Effect.gen(function* () { tab.relay = null; tab.debuggee.debugger.off("message", tab.onMessage); tab.debuggee.webContents.off("did-navigate", tab.onNewDocument); + syncPainting(tab); emit({ type: "detached", ...key }); }; @@ -188,6 +413,14 @@ export const make = Effect.gen(function* () { downloadDirectory: null, pendingDownloadGuid: null, agentInputAt: Number.NEGATIVE_INFINITY, + focusEmulated: false, + focusChange: null, + focusUses: 0, + focusPending: new Map(), + painting: false, + agentUsed: false, + focusReleasing: false, + screencasts: new Set(), onMessage: (_event, method, params, sessionId) => { if (method === "Browser.downloadWillBegin") { const guid = (params as { guid?: unknown } | undefined)?.guid; @@ -198,6 +431,8 @@ export const make = Effect.gen(function* () { // A screenshot pending when the page loads a new document, a reload included, can // stay unanswered. A crashed page lands here once it reloads, well after its renderer died. onNewDocument: () => { + // Before the next screenshot starts, so its own hold survives. + dropFocusHolds(tab); if (tab.screenshotTurn !== null) nextScreenshot(tab); }, }; @@ -211,6 +446,11 @@ export const make = Effect.gen(function* () { Effect.sync(() => { const command = decodeCommand(line); if (Option.isNone(command)) return; + if (command.value.type === "turn") { + if (command.value.active) agentTurns.add(command.value.threadId); + else endAgentTurn(command.value.threadId); + return; + } const tab = tabs.get(keyOf(command.value)); if (!tab) return; if (command.value.type === "pointer") { @@ -219,7 +459,7 @@ export const make = Effect.gen(function* () { return; } if (command.value.type === "release") { - tab.relay = null; + dropRelay(tab); return; } if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now(); @@ -231,19 +471,22 @@ export const make = Effect.gen(function* () { return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_DOWNLOAD_WINDOW_MS; }; - const announceAll = Effect.suspend(() => - Effect.forEach( + const announceAll = Effect.suspend(() => { + // A new backend: the old one's turns ended without telling us. + agentTurns.clear(); + return Effect.forEach( [...tabs.values()], (tab) => { - tab.relay = null; + dropRelay(tab); return PubSub.publish(outbox, { type: "attached", ...tab.key }); }, { discard: true }, - ), - ); + ); + }); return DesktopBrowserHost.of({ pointers: Stream.fromPubSub(pointers), + painting: Stream.fromPubSub(painting), // Subscribes before announcing, so no attach falls between the two. events: Stream.unwrap( Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 3599e7d3cef28..f19079e46bce4 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1713,6 +1713,65 @@ describe("PreviewManager", () => { }).pipe(Effect.provide(managerLayer()), Effect.scoped), ); + effectIt.effect( + "asks the client to paint a navigated server tab while the agent captures it", + () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.DesktopBrowserHost; + const serverTab = { threadId: "thread-1", tabId: "server-tab-1" }; + const contents = Object.assign( + makeTestPreviewWebContents(async () => { + throw new Error("unused"); + }, 42), + { + isDevToolsOpened: () => false, + getUserAgent: () => "Electron", + loadURL: vi.fn(async () => undefined), + }, + ); + const screenshot = Promise.withResolvers(); + contents.debugger.sendCommand = vi.fn((method: string) => + method === "Page.captureScreenshot" ? screenshot.promise : Promise.resolve(undefined), + ) as unknown as typeof contents.debugger.sendCommand; + fromId.mockReturnValue(contents); + const manager = yield* PreviewManager.PreviewManager; + const capturing = yield* Queue.unbounded(); + yield* manager.subscribeStateChanges((_tabId, state) => + Queue.offer(capturing, state.capturing).pipe(Effect.asVoid), + ); + const attached = yield* Deferred.make(); + yield* host.events.pipe( + Stream.runForEach((line) => + new TextDecoder().decode(line).includes('"type":"attached"') + ? Deferred.succeed(attached, undefined) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* manager.createTab("tab_1", { serverTab }); + yield* manager.registerWebview("tab_1", 42); + yield* Deferred.await(attached); + // Navigating rebuilds the tab's state, which must keep the server tab it answers for. + yield* manager.navigate("tab_1", "https://example.com/next"); + yield* Queue.clear(capturing); + + yield* host.handleCommandLine( + JSON.stringify({ + type: "cdp", + ...serverTab, + message: JSON.stringify({ + id: 1, + method: "Page.captureScreenshot", + sessionId: "t3-preview-page", + }), + }), + ); + expect(yield* Queue.take(capturing)).toBe(true); + screenshot.resolve({ data: "" }); + expect(yield* Queue.take(capturing)).toBe(false); + }).pipe(Effect.provide(managerLayer()), Effect.scoped), + ); + effectIt.effect("discards a screenshot that resolves after its guest is replaced", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index b9e0192ca8a7d..d0221e9be0e6e 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -102,6 +102,7 @@ export interface PreviewTabState { audioMuted: boolean; audible: boolean; controller: "human" | "agent" | "none"; + capturing: boolean; favicon?: DesktopPreviewFavicon; serverTab?: { readonly threadId: string; readonly tabId: string }; updatedAt: string; @@ -1652,6 +1653,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function audioMuted: false, audible: false, controller: "none", + capturing: false, updatedAt, }; return [ @@ -1725,6 +1727,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function audioMuted: false, audible: false, controller: "none", + capturing: false, updatedAt, }; yield* emit(tabId, closed); @@ -1946,7 +1949,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function audioMuted: current?.audioMuted ?? false, audible: current?.audible ?? false, controller: current?.controller ?? "none", + capturing: current?.capturing ?? false, ...(current?.favicon ? { favicon: current.favicon } : {}), + ...(current?.serverTab ? { serverTab: current.serverTab } : {}), updatedAt, }; return [ @@ -2299,7 +2304,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return; } if (afterAttach.serverTab) { - yield* listenForAgentPointers; + yield* listenToBrowserHost; browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger }); } if (afterAttach.colorScheme !== "system") { @@ -3239,14 +3244,33 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { discard: true }, ); }); - let pointersStarted = false; - const listenForAgentPointers = Effect.suspend(() => { - if (pointersStarted) return Effect.void; - pointersStarted = true; - return browserHost.pointers.pipe( - Stream.runForEach(emitAgentPointer), - Effect.forkIn(parentScope), - Effect.asVoid, + const updateServerTabCapturing = Effect.fn("PreviewManager.updateServerTabCapturing")( + function* (painting: { + readonly key: { readonly threadId: string; readonly tabId: string }; + readonly active: boolean; + }) { + const tab = [...(yield* SynchronizedRef.get(tabsRef)).values()].find( + (candidate) => + candidate.serverTab?.threadId === painting.key.threadId && + candidate.serverTab.tabId === painting.key.tabId, + ); + if (tab && tab.capturing !== painting.active) { + yield* update(tab.tabId, { capturing: painting.active }); + } + }, + ); + let browserHostStarted = false; + // Subscribes before the attach that follows, so no event falls between the two. + const listenToBrowserHost = Effect.suspend(() => { + if (browserHostStarted) return Effect.void; + browserHostStarted = true; + const listen = Effect.forkIn(parentScope, { startImmediately: true }); + return Effect.all( + [ + browserHost.pointers.pipe(Stream.runForEach(emitAgentPointer), listen), + browserHost.painting.pipe(Stream.runForEach(updateServerTabCapturing), listen), + ], + { discard: true }, ); }); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 1728e3a83ab03..394cc1527cb7d 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -46,6 +46,7 @@ import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter" import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; +import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts"; import * as RunExecutionService from "./RunExecutionService.ts"; import * as RunFinalizationService from "./RunFinalizationService.ts"; import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; @@ -65,6 +66,7 @@ const layerRunExecutionTest = RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({}), @@ -878,6 +880,7 @@ it.effect("passes the thread's MCP app context to the provider under a safe key" RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( + PreviewAgentActivity.layer, Layer.mock(McpAppModelContext.McpAppModelContext)({ forThread: () => Effect.succeed([ @@ -919,6 +922,7 @@ it.effect("fails the run when its ownership check cannot be read before calling Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({ @@ -1120,6 +1124,7 @@ it.effect("starts the provider when checkpoint baseline capture fails", () => Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.fail( @@ -1239,6 +1244,7 @@ it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as co Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => scenario === "start-guard" ? Effect.void : Effect.die("not reached"), @@ -1410,6 +1416,7 @@ it.effect("ingests new member threads of an owned child after the root turn term Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({ @@ -1658,6 +1665,7 @@ it.effect("ends a nested native subagent and its thread when the root run is int Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({ @@ -1851,6 +1859,46 @@ it.effect.each([null, "run:nested-sdk"] as const)( }), ); +// The handover event is not this run's, yet it is what settles the run's agent turns. +it.effect( + "ends preview activity when a transferred child settles a run kept open by background work", + () => + Effect.gen(function* () { + const active = yield* Deferred.make(); + yield* runBackgroundItemScenario( + "child-transfer-preview", + (ids) => { + const running = subagentEvent(ids, "running"); + if (running.type !== "subagent.updated") throw new Error("Missing subagent"); + const linked = { + ...running, + subagent: { + ...running.subagent, + origin: "provider_native" as const, + childThreadId: ids.childThreadId, + providerThreadId: null, + }, + }; + return [ + childThreadCreatedEvent(ids), + linked, + backgroundTurnItemEvent(ids, "command_execution", "running", 1), + rootTerminalEvent(ids, "completed"), + { ...linked, subagent: { ...linked.subagent, runId: RunId.make("run:replacement") } }, + ]; + }, + { + keepEventStreamOpen: true, + afterEvents: (activity) => + activity + .active(backgroundScenarioIds("child-transfer-preview").threadId) + .pipe(Effect.flatMap((value) => Deferred.succeed(active, value))), + }, + ).pipe(Effect.forkScoped); + assert.isFalse(yield* Deferred.await(active)); + }).pipe(Effect.scoped), +); + it.effect("keeps ingesting a failed root's background item when its work continues", () => Effect.gen(function* () { const observed = yield* runBackgroundItemScenario("bg-failed-continues", (ids) => [ @@ -1960,6 +2008,7 @@ it.effect( Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void, }), @@ -2769,6 +2818,95 @@ it.effect("does not interrupt a provider when ingestion fails before the root tu }), ); +// The server browser keeps an agent's tab focused for exactly this span. +it.effect.each(["completed", "failed"] as const)( + "reports the thread executing until its %s run's event consumer ends", + (outcome) => + Effect.gen(function* () { + const gate = yield* Deferred.make(); + const { previewActivityEnded } = yield* captureRootRunTermination({ + key: `preview-activity:${outcome}`, + shouldFinalizeRun: () => Effect.succeed(true), + events: (ids) => + Stream.fromEffect(Deferred.await(gate)).pipe( + Stream.flatMap(() => + outcome === "completed" + ? Stream.make(rootTerminalEvent(ids, "completed")) + : Stream.fail( + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver, + providerSessionId: ProviderSessionId.make("session:exited"), + cause: "provider process exited", + }), + ), + ), + ), + afterStart: () => + Effect.gen(function* () { + const activity = yield* PreviewAgentActivity.PreviewAgentActivity; + assert.isTrue( + yield* activity.active(ThreadId.make(`thread:preview-activity:${outcome}`)), + ); + yield* Deferred.succeed(gate, undefined); + }), + }); + assert.isTrue(previewActivityEnded); + }), +); + +// Background commands keep the consumer alive for the whole session; only the agent's own +// turn and the children it drives should hold the browser tab focused. +it.effect.each([ + { name: "ends once the root turn finishes beside background work", openSubagent: false }, + { name: "stays while a run-owned subagent continues", openSubagent: true }, +])("preview activity $name", ({ openSubagent }) => + Effect.gen(function* () { + const key = `preview-activity:background:${openSubagent}`; + const consumerWaiting = yield* Deferred.make(); + const release = yield* Deferred.make(); + const providerInstanceId = ProviderInstanceId.make("codex"); + yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + hasPendingBackgroundWorkForThread: () => Effect.succeed(true), + events: (ids) => + Stream.concat( + Stream.fromIterable([ + ...(openSubagent + ? [ + { + type: "subagent.updated", + driver, + subagent: makeRunOwnedSubagentFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + }), + } as const, + ] + : []), + rootTerminalEvent(ids, "completed"), + ] satisfies ReadonlyArray), + Stream.fromEffect( + Deferred.succeed(consumerWaiting, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + ).pipe(Stream.drain), + ), + afterStart: (closed) => + Effect.gen(function* () { + yield* Deferred.await(consumerWaiting); + assert.isFalse(yield* Deferred.isDone(closed)); + const activity = yield* PreviewAgentActivity.PreviewAgentActivity; + assert.equal(yield* activity.active(ThreadId.make(`thread:${key}`)), openSubagent); + yield* Deferred.succeed(release, undefined); + }), + }); + }), +); + it.effect("fails the run when its event stream ends without a root terminal", () => Effect.gen(function* () { const { observed, finalEvents } = yield* captureRootRunTermination({ @@ -2838,6 +2976,7 @@ function captureRootRunTermination(input: { readonly shouldFinalizeRun: () => Effect.Effect; readonly rejectTerminalWrite?: boolean; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; + readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"]; readonly seedOpenSubagent?: boolean; readonly events?: ( ids: BackgroundScenarioIds, @@ -2859,7 +2998,9 @@ function captureRootRunTermination(input: { readonly beforeTerminalWrite?: ( request: Parameters[0], ) => Effect.Effect; - readonly afterStart?: (closed: Deferred.Deferred) => Effect.Effect; + readonly afterStart?: ( + closed: Deferred.Deferred, + ) => Effect.Effect; }) { return Effect.gen(function* () { const ids = backgroundScenarioIds(input.key); @@ -2946,9 +3087,10 @@ function captureRootRunTermination(input: { }), ), ), + Layer.provideMerge(PreviewAgentActivity.layer), ); - yield* Effect.gen(function* () { + const previewActivityEnded = yield* Effect.gen(function* () { const runExecution = yield* RunExecutionService.RunExecutionServiceV2; const start = runExecution.startRootRun({ commandId: CommandId.make(`command:${input.key}`), @@ -2989,6 +3131,7 @@ function captureRootRunTermination(input: { }), startTurn: input.startTurn ?? (() => Effect.void), interruptTurn: input.interruptTurn ?? (() => Effect.void), + hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread, } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime, run: { id: ids.runId, @@ -3050,9 +3193,12 @@ function captureRootRunTermination(input: { ); yield* input.afterStart?.(ingestionDone) ?? Effect.void; yield* Deferred.await(ingestionDone); + const activity = yield* PreviewAgentActivity.PreviewAgentActivity; + return !(yield* activity.active(ids.threadId)); }).pipe(Effect.provide(layerTest)); return { + previewActivityEnded, written: yield* Ref.get(writtenItems), finalEvents: yield* Ref.get(finalEvents), observed: yield* Ref.get(observed), @@ -3387,6 +3533,9 @@ function runBackgroundItemScenario( >; readonly onSubscribe?: Effect.Effect; readonly onEventsConsumed?: Effect.Effect; + readonly afterEvents?: ( + activity: PreviewAgentActivity.PreviewAgentActivity["Service"], + ) => Effect.Effect; readonly onFinalized?: ( events: ReadonlyArray, ) => Effect.Effect; @@ -3397,10 +3546,14 @@ function runBackgroundItemScenario( const providerInstanceId = ProviderInstanceId.make("codex"); const observed = yield* Ref.make>([]); const ingestionDone = yield* Deferred.make(); + const previewActivity = yield* PreviewAgentActivity.PreviewAgentActivity.pipe( + Effect.provide(PreviewAgentActivity.layer), + ); const layerTest = RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + Layer.succeed(PreviewAgentActivity.PreviewAgentActivity, previewActivity), Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, Layer.mock(EventSink.EventSinkV2)({ @@ -3456,7 +3609,12 @@ function runBackgroundItemScenario( yield* options?.onSubscribe ?? Effect.void; const events = Stream.fromIterable(makeEvents(ids)).pipe( Stream.concat( - Stream.fromEffect(options?.onEventsConsumed ?? Effect.void).pipe(Stream.drain), + Stream.fromEffect( + Effect.andThen( + options?.onEventsConsumed ?? Effect.void, + options?.afterEvents?.(previewActivity) ?? Effect.void, + ), + ).pipe(Stream.drain), ), ); return { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index d7789df380b68..7878befe0e852 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -39,6 +39,7 @@ import * as Stream from "effect/Stream"; import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; +import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as CheckpointService from "./CheckpointService.ts"; import * as EventSink from "./EventSink.ts"; @@ -595,6 +596,7 @@ export const layer: Layer.Layer< | ProviderEventIngestor.ProviderEventIngestorV2 | ServerSettings.ServerSettingsService | McpAppModelContext.McpAppModelContext + | PreviewAgentActivity.PreviewAgentActivity | TurnItemPositionStore.TurnItemPositionStoreV2 > = Layer.effect( RunExecutionServiceV2, @@ -606,6 +608,7 @@ export const layer: Layer.Layer< const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; const serverSettings = yield* ServerSettings.ServerSettingsService; const mcpAppModelContext = yield* McpAppModelContext.McpAppModelContext; + const previewAgentActivity = yield* PreviewAgentActivity.PreviewAgentActivity; const turnItemPositions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; const finalizationObserver = yield* RunFinalizationService.RunFinalizationObserver; @@ -1220,7 +1223,7 @@ export const layer: Layer.Layer< } } }); - const shouldStopProviderEventIngestion = Effect.gen(function* () { + const agentTurnsSettled = Effect.gen(function* () { if (!(yield* Ref.get(rootTerminalSeen))) { return false; } @@ -1237,9 +1240,20 @@ export const layer: Layer.Layer< return false; } const childSubagents = yield* Ref.get(activeChildSubagents); - if (childSubagents.size > 0) { + return childSubagents.size === 0; + }); + const shouldStopProviderEventIngestion = Effect.gen(function* () { + if (!(yield* agentTurnsSettled)) { return false; } + const terminal = yield* Ref.get(terminalEvent); + if ( + terminal !== null && + terminal.status !== "completed" && + !childTurnsContinue(terminal) + ) { + return true; + } // Newly owned items depend on adapters emitting a non-terminal event before the root // terminal. Owner loss (a newer run claimed lastRunOrdinal) must not close this stream // while these sets are non-empty: turn_item.updated writes are not ownership-gated, so @@ -1269,6 +1283,9 @@ export const layer: Layer.Layer< // Held text writes nothing, so its item would take a position only // when finally written, behind every item that began after it. const reservedHeldTurnItemIds = new Set(); + // Ends once the root turn and the children it drives settle. Background commands such + // as a dev server keep this consumer alive afterwards and must not pin preview focus. + const endPreviewActivity = yield* previewAgentActivity.begin(input.run.threadId); const providerEventFiber = yield* eventSubscription.events.pipe( Stream.mapEffect((event) => Effect.gen(function* () { @@ -1444,6 +1461,12 @@ export const layer: Layer.Layer< yield* trackChildLifecycle(event, deliveredEvent !== null); }), ), + // Also after events this run does not accept, such as a child handed to another run. + Stream.tap(() => + Effect.gen(function* () { + if (yield* agentTurnsSettled) yield* endPreviewActivity; + }), + ), Stream.takeUntilEffect(() => shouldStopProviderEventIngestion), Stream.runDrain, Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })), @@ -1574,6 +1597,7 @@ export const layer: Layer.Layer< ); }), ), + Effect.ensuring(endPreviewActivity), Effect.ensuring(eventSubscription.close), Effect.forkDetach, ); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index 8e1c65443ea3c..eb6f9d1d3eaa1 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -6,6 +6,7 @@ import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as OrchestrationCommandReceipts from "../persistence/OrchestrationCommandReceipts.ts"; import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts"; import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; +import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts"; import * as McpAppRequests from "../mcpApps/McpAppRequests.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; import * as TextGeneration from "../textGeneration/TextGeneration.ts"; @@ -153,6 +154,7 @@ const layerRunExecutionServiceProvided = RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( McpAppModelContext.layer, + PreviewAgentActivity.layer, layerCheckpointServiceProvided, layerEventSinkProvided, IdAllocator.layer, diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 76f89af4847d1..be25176dc3f1b 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -48,6 +48,7 @@ import * as ProviderTurnControlService from "../ProviderTurnControlService.ts"; import * as ProviderTurnStartService from "../ProviderTurnStartService.ts"; import * as ProviderTurnStartServiceTestkit from "../ProviderTurnStartService.testkit.ts"; import * as McpAppModelContext from "../../mcpApps/McpAppModelContext.ts"; +import * as PreviewAgentActivity from "../../preview/PreviewAgentActivity.ts"; import * as RunExecutionService from "../RunExecutionService.ts"; import * as RunFinalizationService from "../RunFinalizationService.ts"; import * as ThreadTitleRegenerationService from "../ThreadTitleRegenerationService.ts"; @@ -378,6 +379,7 @@ export function layerWithRegistry( Layer.provide( Layer.mergeAll( McpAppModelContext.layerEmpty, + PreviewAgentActivity.layer, layerCheckpointServiceProvided, layerEventSinkProvided, IdAllocator.layer, diff --git a/apps/server/src/preview/DesktopBrowserChannel.ts b/apps/server/src/preview/DesktopBrowserChannel.ts index 4d8ab5e098c3e..d56b2a10a542b 100644 --- a/apps/server/src/preview/DesktopBrowserChannel.ts +++ b/apps/server/src/preview/DesktopBrowserChannel.ts @@ -49,6 +49,8 @@ export class DesktopBrowserChannel extends Context.Service< key: DesktopTabKey, pointer: { readonly phase: "move" | "click"; readonly x: number; readonly y: number }, ) => Effect.Effect; + /** Never fails. A failed write means a broken channel; the next backend starts with no turns. */ + readonly turn: (threadId: string, active: boolean) => Effect.Effect; } >()("t3/preview/DesktopBrowserChannel") {} @@ -70,6 +72,7 @@ const make = Effect.gen(function* () { isAttached: () => Effect.succeed(false), endpoint: () => Effect.die("No desktop app is attached to this server."), pointer: () => Effect.void, + turn: () => Effect.void, }); } @@ -207,6 +210,7 @@ const make = Effect.gen(function* () { isAttached: (key) => Effect.sync(() => attachedTabs.has(keyOf(key))), endpoint, pointer: (key, pointer) => command({ type: "pointer", ...key, ...pointer }), + turn: (threadId, active) => command({ type: "turn", threadId, active }), }); }); diff --git a/apps/server/src/preview/PreviewAgentActivity.ts b/apps/server/src/preview/PreviewAgentActivity.ts new file mode 100644 index 0000000000000..2e7d261de9e57 --- /dev/null +++ b/apps/server/src/preview/PreviewAgentActivity.ts @@ -0,0 +1,58 @@ +import type { ThreadId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import type * as Scope from "effect/Scope"; + +export interface PreviewAgentTurnChange { + readonly threadId: ThreadId; + readonly active: boolean; +} + +/** + * Whether an agent turn runs on a thread, model thinking included, so its preview tab + * stays focused between tool calls. Background commands outlive the turn and do not count. + */ +export class PreviewAgentActivity extends Context.Service< + PreviewAgentActivity, + { + /** Marks the thread active until the returned effect runs; it runs at most once. */ + readonly begin: (threadId: ThreadId) => Effect.Effect>; + readonly active: (threadId: ThreadId) => Effect.Effect; + /** Each thread's transitions between no turn and some turn, from subscription on. */ + readonly changes: Effect.Effect< + PubSub.Subscription, + never, + Scope.Scope + >; + } +>()("t3/preview/PreviewAgentActivity") {} + +const make = Effect.gen(function* () { + const changes = yield* PubSub.unbounded(); + // A token per turn, so overlapping turns on one thread end only when the last does. + const turns = new Map>(); + + const begin = Effect.fn("PreviewAgentActivity.begin")(function* (threadId: ThreadId) { + const existing = turns.get(threadId); + const tokens = existing ?? new Set(); + if (existing === undefined) turns.set(threadId, tokens); + const token = Symbol(threadId); + tokens.add(token); + if (existing === undefined) yield* PubSub.publish(changes, { threadId, active: true }); + return Effect.suspend(() => { + if (!tokens.delete(token) || tokens.size > 0) return Effect.void; + if (turns.get(threadId) === tokens) turns.delete(threadId); + return PubSub.publish(changes, { threadId, active: false }).pipe(Effect.asVoid); + }); + }); + + return PreviewAgentActivity.of({ + begin, + active: (threadId) => Effect.sync(() => turns.has(threadId)), + changes: PubSub.subscribe(changes), + }); +}); + +export const layer = Layer.effect(PreviewAgentActivity, make); diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 0a9118e7b6d8b..c8061833a80ec 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -32,6 +32,7 @@ import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as Broker from "../mcp/PreviewAutomationBroker.ts"; import * as DesktopChannel from "./DesktopBrowserChannel.ts"; import * as Manager from "./Manager.ts"; +import * as PreviewAgentActivity from "./PreviewAgentActivity.ts"; import * as ServerBrowser from "./ServerBrowser.ts"; import * as PreviewBrowser from "./PreviewBrowser.ts"; @@ -275,6 +276,7 @@ const desktopRenders = (tabId: string) => { return desktopTabs.has(tabId); }; const releasedDesktopTabs: Array = []; +const desktopTurns = new NodeEvents.EventEmitter(); const desktopConnections: Array<{ endpoint: string; context: ReturnType }> = []; const testThread = { threadId: ThreadId.make("browser-test-thread"), @@ -335,7 +337,9 @@ const dependencies = Layer.mergeAll( Effect.sync(() => releasedDesktopTabs.push(key.tabId)), ), pointer: () => Effect.void, + turn: (threadId, active) => Effect.sync(() => desktopTurns.emit("turn", threadId, active)), }), + PreviewAgentActivity.layer, ).pipe( Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-server-browser-" })), Layer.provideMerge(NodeServices.layer), @@ -378,6 +382,28 @@ beforeEach(() => { desktopConnections.length = 0; }); +it.live("tells the desktop once per thread when its agent turns start and end", () => + Effect.gen(function* () { + yield* ServerBrowser.ServerBrowser; + const activity = yield* PreviewAgentActivity.PreviewAgentActivity; + const threadId = scope.thread.threadId; + const turns = yield* Queue.unbounded(); + const record = (id: string, active: boolean) => Queue.offerUnsafe(turns, [id, active]); + desktopTurns.on("turn", record); + yield* Effect.addFinalizer(() => Effect.sync(() => desktopTurns.off("turn", record))); + const endFirst = yield* activity.begin(threadId); + // An overlapping turn on the same thread neither starts nor ends focus on its own. + const endSecond = yield* activity.begin(threadId); + expect(yield* Queue.take(turns)).toEqual([threadId, true]); + yield* endFirst; + yield* endFirst; + expect(yield* activity.active(threadId)).toBe(true); + yield* endSecond; + expect(yield* Queue.take(turns)).toEqual([threadId, false]); + expect(yield* Queue.size(turns)).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + it.live("preview_open reports its navigation timeout instead of the request budget", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 574f843b1991c..93f6849bbb995 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -77,6 +77,7 @@ import { resolveRootCliCommand } from "../cli/invocation.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as PreviewAutomationBroker from "../mcp/PreviewAutomationBroker.ts"; import * as DesktopBrowserChannel from "./DesktopBrowserChannel.ts"; +import * as PreviewAgentActivity from "./PreviewAgentActivity.ts"; import * as PreviewManager from "./Manager.ts"; import * as ServerBrowserPage from "./ServerBrowserPage.ts"; import * as PreviewBrowser from "./PreviewBrowser.ts"; @@ -454,6 +455,7 @@ const make = Effect.gen(function* () { const environment = yield* ServerEnvironment.ServerEnvironment; const previewBrowser = yield* PreviewBrowser.PreviewBrowser; const desktopChannel = yield* DesktopBrowserChannel.DesktopBrowserChannel; + const agentTurns = yield* (yield* PreviewAgentActivity.PreviewAgentActivity).changes; const runFork = Effect.runForkWith(yield* Effect.context()); const launchServices = yield* Effect.context(); const setupCommand = yield* resolveRootCliCommand(PreviewBrowserHost.SETUP_SUBCOMMAND); @@ -2458,6 +2460,11 @@ const make = Effect.gen(function* () { ), Effect.forkScoped, ); + // The desktop keeps the tabs an agent uses focused until its turn ends, not just per command. + yield* Stream.fromSubscription(agentTurns).pipe( + Stream.runForEach((change) => desktopChannel.turn(change.threadId, change.active)), + Effect.forkScoped, + ); // Reconnect returned pages immediately so all clients keep receiving URL and title changes. yield* desktopChannel.attached.pipe( Stream.runForEach((key) => diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 6cde73143a048..f461a99186b31 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -74,6 +74,7 @@ import * as DeviceHubProxy from "./device/DeviceHubProxy.ts"; import * as PreviewManager from "./preview/Manager.ts"; import * as PortScanner from "./preview/PortScanner.ts"; import * as ServerBrowser from "./preview/ServerBrowser.ts"; +import * as PreviewAgentActivity from "./preview/PreviewAgentActivity.ts"; import * as DesktopBrowserChannel from "./preview/DesktopBrowserChannel.ts"; import * as ServerBrowserStream from "./preview/ServerBrowserStream.ts"; import * as PreviewBrowser from "./preview/PreviewBrowser.ts"; @@ -732,7 +733,12 @@ const layerMakeRoutes = Layer.mergeAll( ), // Share the service so HTTP/WS clients single-flight reads and invalidate each other's cached patches. Layer.provide(layerPullRequestService), - Layer.provide(ServerBrowser.layer.pipe(Layer.provide(DesktopBrowserChannel.layer))), + // The same activity instance RunExecutionService reports to, by layer memoization. + Layer.provide( + ServerBrowser.layer.pipe( + Layer.provide(Layer.merge(DesktopBrowserChannel.layer, PreviewAgentActivity.layer)), + ), + ), Layer.provide(PreviewBrowser.layer), Layer.provide(PreviewAutomationBroker.layer), Layer.provide(ServerSelfUpdate.layer.pipe(Layer.provide(layerDesktopAppUpdate))), diff --git a/apps/web/src/components/preview/usePreviewBridge.test.ts b/apps/web/src/components/preview/usePreviewBridge.test.ts index 14acf0d69609e..736ac7d743c32 100644 --- a/apps/web/src/components/preview/usePreviewBridge.test.ts +++ b/apps/web/src/components/preview/usePreviewBridge.test.ts @@ -1,7 +1,9 @@ import type { DesktopPreviewTabState } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; -import { projectDesktopState } from "./usePreviewBridge"; +import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; + +import { createCaptureActivity, projectDesktopState } from "./usePreviewBridge"; const favicon = { dataUrl: "data:image/png;base64,AAAA", @@ -22,6 +24,7 @@ function state(navStatus: DesktopPreviewTabState["navStatus"]): DesktopPreviewTa audioMuted: false, audible: false, controller: "none", + capturing: false, favicon, updatedAt: "2026-08-09T00:00:00.000Z", }; @@ -48,3 +51,23 @@ describe("projectDesktopState", () => { expect(projectDesktopState(state({ kind: "Idle" })).favicon).toBeNull(); }); }); + +describe("createCaptureActivity", () => { + it("keeps the guest painting while the host captures it, alongside other activity", () => { + const activity = () => useBrowserSurfaceStore.getState().activityByTabId["tab-1"]; + const recording = useBrowserSurfaceStore.getState().acquireActivity("tab-1"); + const capture = createCaptureActivity("tab-1"); + + capture.update(true); + capture.update(true); + expect(activity()).toBe(2); + capture.update(false); + expect(activity()).toBe(1); + capture.update(true); + capture.release(); + capture.release(); + expect(activity()).toBe(1); + recording(); + expect(activity()).toBeUndefined(); + }); +}); diff --git a/apps/web/src/components/preview/usePreviewBridge.ts b/apps/web/src/components/preview/usePreviewBridge.ts index a58382d4f83e8..69c6be2046fea 100644 --- a/apps/web/src/components/preview/usePreviewBridge.ts +++ b/apps/web/src/components/preview/usePreviewBridge.ts @@ -8,6 +8,7 @@ import { useEffect, useEffectEvent, useRef } from "react"; import { recordFaviconForThread } from "~/browserFaviconStore"; import { useBrowserPointerStore } from "~/browser/browserPointerStore"; +import { acquireBrowserSurfaceActivity } from "~/browser/browserSurfaceStore"; import { applyPreviewDesktopState, type DesktopPreviewOverlay } from "~/previewStateStore"; import { previewEnvironment } from "~/state/preview"; import { useAtomCommand } from "~/state/use-atom-command"; @@ -22,6 +23,22 @@ function originOf(url: string): string | null { } } +/** + * Keeps a hidden guest painting behind the app while the desktop host captures it; Electron + * composites no offscreen guest, so its screenshot or stream would otherwise never finish. + */ +export function createCaptureActivity(tabId: string) { + let release: (() => void) | null = null; + const update = (capturing: boolean) => { + if (capturing && !release) release = acquireBrowserSurfaceActivity(tabId); + else if (!capturing && release) { + release(); + release = null; + } + }; + return { update, release: () => update(false) }; +} + /** ElectronBrowserHost owns webview lifetime. */ export function usePreviewBridge(input: { threadRef: ScopedThreadRef; @@ -38,6 +55,7 @@ export function usePreviewBridge(input: { const lastReportedUrl = useRef(null); const lastReportedKind = useRef(null); const lastDesktopNavStatus = useRef(null); + const captureActivity = useRef | null>(null); const handleStateChange = useEffectEvent( (changedTabId: string, state: DesktopPreviewTabState): void => { if (changedTabId !== runtimeTabId) return; @@ -45,6 +63,7 @@ export function usePreviewBridge(input: { clearBrowserPointer(runtimeTabId); } lastDesktopNavStatus.current = state.navStatus; + captureActivity.current?.update(state.capturing); applyPreviewDesktopState(stableThreadRef, tabId, projectDesktopState(state)); if (state.favicon) { recordFaviconForThread(stableThreadRef, state.favicon); @@ -70,11 +89,19 @@ export function usePreviewBridge(input: { lastReportedUrl.current = null; lastReportedKind.current = null; lastDesktopNavStatus.current = null; - return previewBridge.onStateChange(handleStateChange); + const activity = createCaptureActivity(runtimeTabId); + captureActivity.current = activity; + const unsubscribe = previewBridge.onStateChange(handleStateChange); + return () => { + unsubscribe(); + activity.release(); + if (captureActivity.current === activity) captureActivity.current = null; + }; }, [ - // oxlint-disable-next-line react/exhaustive-effect-dependencies -- A different tab or thread starts its status reports and pointer tracking from scratch. runtimeTabId, - stableThreadRef, + // oxlint-disable-next-line react/exhaustive-effect-dependencies -- A different tab or thread starts its status reports and pointer tracking from scratch. + environmentId, + threadId, tabId, ]); } diff --git a/packages/contracts/src/desktopBrowser.ts b/packages/contracts/src/desktopBrowser.ts index c70b374910079..17d256f0d8d3e 100644 --- a/packages/contracts/src/desktopBrowser.ts +++ b/packages/contracts/src/desktopBrowser.ts @@ -25,5 +25,11 @@ export const DesktopBrowserCommand = Schema.Union([ x: Schema.Finite, y: Schema.Finite, }), + /** Whether an agent turn runs on the thread, so the tabs it uses stay focused between steps. */ + Schema.Struct({ + type: Schema.Literal("turn"), + threadId: TrimmedNonEmptyString, + active: Schema.Boolean, + }), ]); export type DesktopBrowserCommand = typeof DesktopBrowserCommand.Type; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 1a91b8a259441..34d4162081751 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -590,6 +590,8 @@ export interface DesktopPreviewTabState { /** Chromium reports playback even when audioMuted is true. */ audible: boolean; controller: "human" | "agent" | "none"; + /** A screenshot or stream of the guest is pending, so the client keeps it painting while hidden. */ + capturing: boolean; favicon?: DesktopPreviewFavicon; updatedAt: string; }