diff --git a/.github/actions/ci-changes/action.yml b/.github/actions/ci-changes/action.yml index ecf787404657a..b531653fab4b7 100644 --- a/.github/actions/ci-changes/action.yml +++ b/.github/actions/ci-changes/action.yml @@ -19,6 +19,8 @@ outputs: value: ${{ steps.detect.outputs.release_smoke }} mobile_native: value: ${{ steps.detect.outputs.mobile_native }} + desktop_preview: + value: ${{ steps.detect.outputs.desktop_preview }} runs: using: composite steps: diff --git a/.github/scripts/ci-changes.cjs b/.github/scripts/ci-changes.cjs index 6a7fc4a6d5b5f..fd32f99e0db17 100644 --- a/.github/scripts/ci-changes.cjs +++ b/.github/scripts/ci-changes.cjs @@ -64,6 +64,11 @@ const BUILD_ONLY_DEPS = { t3: ["@t3tools/web"] }; const MOBILE_NATIVE = /^apps\/mobile\/.*\.(swift|kt|kts)$|^apps\/mobile\/(\.swiftlint\.yml|detekt\.yml|\.editorconfig|Brewfile)$|^scripts\/mobile-native-static-check\.ts$|^package\.json$|^\.github\/workflows\/ci\.yml$|^\.github\/scripts\/ci-changes\.cjs$|^\.github\/actions\/ci-changes\//; +// Real-Electron check that agent preview tabs paint with the app window hidden. +// Electron is pinned in the desktop manifest, so its upgrades rerun it too. +const DESKTOP_PREVIEW = + /^apps\/desktop\/src\/preview\/|^apps\/desktop\/scripts\/verify-background-preview|^apps\/desktop\/package\.json$|^\.github\/workflows\/ci\.yml$|^\.github\/scripts\/ci-changes\.cjs$|^\.github\/actions\/ci-changes\//; + // Jobs that test every package except these, which have jobs of their own. const OWN_TEST_JOB = new Set(["t3", "@t3tools/web", "@t3tools/monorepo"]); @@ -181,6 +186,7 @@ function decide(files, packages, full) { rust: everything || rust, release_smoke: scriptsChanged, mobile_native: files === null || files.some((file) => MOBILE_NATIVE.test(file)), + desktop_preview: files === null || files.some((file) => DESKTOP_PREVIEW.test(file)), }, }; } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d29d970c75ff1..d600877437794 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: rust: ${{ steps.detect.outputs.rust }} release_smoke: ${{ steps.detect.outputs.release_smoke }} mobile_native: ${{ steps.detect.outputs.mobile_native }} + desktop_preview: ${{ steps.detect.outputs.desktop_preview }} steps: # Only the package manifests the script reads to build the dependency graph. # Depth 2 brings the base commit the change is diffed against. @@ -470,6 +471,38 @@ jobs: - name: Lint mobile native sources run: vp run lint:mobile + # Unit tests mock Electron, so only a real one with a real window manager can + # prove agent preview tabs still paint while the app window is hidden. + desktop_preview: + name: Desktop Background Preview + needs: changes + if: ${{ !cancelled() && needs.changes.outputs.desktop_preview != 'false' }} + runs-on: blacksmith-6vcpu-macos-26 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: true + run-install: | + args: + - --filter=@t3tools/desktop + + - name: Ensure Electron runtime is installed + run: vp run --filter @t3tools/desktop ensure:electron + + - name: Verify background preview tabs paint + run: node apps/desktop/scripts/verify-background-preview.mjs + release_smoke: name: Release Smoke needs: changes @@ -510,6 +543,7 @@ jobs: test_server, rust, mobile_native_static_analysis, + desktop_preview, release_smoke, ] runs-on: blacksmith-2vcpu-ubuntu-2404 diff --git a/apps/desktop/scripts/verify-background-preview.electron.cjs b/apps/desktop/scripts/verify-background-preview.electron.cjs new file mode 100644 index 0000000000000..54a847a0189a0 --- /dev/null +++ b/apps/desktop/scripts/verify-background-preview.electron.cjs @@ -0,0 +1,94 @@ +// Electron entry for verify-background-preview.mjs. CommonJS, because an ESM +// entry delays `ready` until its top-level await settles. +const NodeFS = require("node:fs"); +const NodeOS = require("node:os"); +const NodePath = require("node:path"); +const NodeURL = require("node:url"); +const { app, BrowserWindow, nativeImage } = require("electron"); + +const CAPTURE_TIMEOUT_MS = 5_000; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const withTimeout = (promise, ms) => + Promise.race([ + promise.then((value) => ({ ok: true, value })), + sleep(ms).then(() => ({ ok: false })), + ]); +const page = (html) => `data:text/html,${encodeURIComponent(html)}`; + +async function captureColor(tab, rgb) { + await tab.debugger.sendCommand("Runtime.evaluate", { + expression: `document.body.style.background = 'rgb(${rgb.join(",")})'`, + }); + const shot = await withTimeout( + tab.debugger.sendCommand("Page.captureScreenshot", { format: "png" }), + CAPTURE_TIMEOUT_MS, + ); + if (!shot.ok) return "timed out"; + const pixels = nativeImage.createFromBuffer(Buffer.from(shot.value.data, "base64")).toBitmap(); + const got = [pixels[2], pixels[1], pixels[0]]; + // Color management can shift a channel by a few levels. + return got.every((value, index) => Math.abs(value - rgb[index]) <= 6) + ? null + : `stale: rgb(${got.join(",")})`; +} + +async function main() { + const { backgroundTabWindowOptions } = await import( + NodeURL.pathToFileURL(NodePath.join(__dirname, "../src/preview/BackgroundTabWindow.ts")).href + ); + const preload = NodePath.join( + NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-background-preview-")), + "preload.cjs", + ); + NodeFS.writeFileSync(preload, ""); + // Each state closes its windows, which would otherwise quit the app. + app.on("window-all-closed", () => {}); + await app.whenReady(); + + const failures = []; + for (const state of ["hidden", "minimized"]) { + const appWindow = new BrowserWindow({ width: 480, height: 320, show: true }); + await appWindow.loadURL(page("T3 Code")); + await sleep(500); + if (state === "hidden") appWindow.hide(); + else appWindow.minimize(); + await sleep(1_500); + + // Created only after the app window is gone, as an agent's tab would be. + const tabWindow = new BrowserWindow( + backgroundTabWindowOptions({ + partition: `t3-background-preview-${state}`, + preload, + passkeyBridgeArgument: undefined, + }), + ); + await tabWindow.loadURL(page("")); + const tab = tabWindow.webContents; + tab.debugger.attach("1.3"); + for (const [step, rgb] of [ + [1, [0, 200, 0]], + [2, [200, 200, 0]], + [3, [0, 200, 200]], + ]) { + const failure = await captureColor(tab, rgb); + if (failure) failures.push(`app window ${state}, capture ${step}: ${failure}`); + await sleep(1_000); + } + tabWindow.destroy(); + appWindow.destroy(); + } + + if (failures.length > 0) { + console.error("Background preview tabs stopped painting:"); + for (const failure of failures) console.error(` - ${failure}`); + app.exit(1); + return; + } + console.log("Background preview tabs paint while the app window is hidden or minimized."); + app.exit(0); +} + +main().catch((error) => { + console.error(error); + app.exit(1); +}); diff --git a/apps/desktop/scripts/verify-background-preview.mjs b/apps/desktop/scripts/verify-background-preview.mjs new file mode 100644 index 0000000000000..1907efb3e4a31 --- /dev/null +++ b/apps/desktop/scripts/verify-background-preview.mjs @@ -0,0 +1,27 @@ +// Proves agent preview tabs still paint while the app window is hidden or +// minimized, using the exact window options the preview manager ships. +// Agents open tabs whenever they like; a tab that cannot paint makes every +// screenshot of it time out. Unit tests mock Electron, so only a real one can +// catch this. +import * as NodeChildProcess from "node:child_process"; +import * as NodeModule from "node:module"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; + +const RUN_TIMEOUT_MS = 90_000; +const entry = NodePath.join( + NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), + "verify-background-preview.electron.cjs", +); +// The plain binary: the development launcher's renamed app bundle refuses data: URLs. +const electronPath = NodeModule.createRequire(import.meta.url)("electron"); +const { ELECTRON_RUN_AS_NODE: _ignored, ...env } = process.env; +const child = NodeChildProcess.spawn(electronPath, [entry], { stdio: "inherit", env }); +const timer = setTimeout(() => { + console.error(`Background preview check did not finish within ${RUN_TIMEOUT_MS}ms.`); + child.kill("SIGKILL"); +}, RUN_TIMEOUT_MS); +child.on("exit", (code, signal) => { + clearTimeout(timer); + process.exit(code ?? (signal ? 1 : 0)); +}); diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index b659054540c23..985d1bfa7177c 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -80,9 +80,25 @@ const createTab = DesktopIpc.makeIpcMethod({ zoomFactor, colorScheme, serverTab, + background, }) { const manager = yield* PreviewManager.PreviewManager; - yield* manager.createTab(tabId, { zoomFactor, colorScheme, serverTab }); + if (background === undefined) { + yield* manager.createTab(tabId, { zoomFactor, colorScheme, serverTab }); + return; + } + const { scope, persistent, namespace } = resolvePartitionScope( + background.environmentId, + background.profileId, + ); + // Loading the session installs its UA rewrite and permission handlers. + yield* manager.getBrowserSession(scope, persistent, namespace); + const partition = yield* manager.getBrowserPartition(scope, persistent, namespace); + yield* manager.createTab( + tabId, + { zoomFactor, colorScheme, serverTab }, + { partition, url: background.url }, + ); }), }); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index bffa02d9ca435..aec27911c40a0 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -327,6 +327,7 @@ contextBridge.exposeInMainWorld("desktopBridge", { zoomFactor: defaults?.zoomFactor, serverTab: defaults?.serverTab, colorScheme: defaults?.colorScheme, + background: defaults?.background, }), closeTab: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_CLOSE_TAB_CHANNEL, { tabId }), registerWebview: (tabId, webContentsId) => diff --git a/apps/desktop/src/preview/BackgroundTabWindow.ts b/apps/desktop/src/preview/BackgroundTabWindow.ts new file mode 100644 index 0000000000000..f9ceb472caf97 --- /dev/null +++ b/apps/desktop/src/preview/BackgroundTabWindow.ts @@ -0,0 +1,42 @@ +import type { BrowserWindowConstructorOptions } from "electron"; + +// Kept free of runtime imports: scripts/verify-background-preview.mjs loads it +// straight into Electron to prove these options still paint while the app +// window is hidden. + +// The server sizes the page through emulation; this only seeds the window. +const BACKGROUND_TAB_WIDTH = 1280; +const BACKGROUND_TAB_HEIGHT = 800; + +/** + * An agent tab's never-shown window. A panel `` that attaches while + * the app window is hidden or minimized never paints, so its screenshots never + * return; a window of its own paints in every app window state. + */ +export function backgroundTabWindowOptions(input: { + readonly partition: string; + readonly preload: string; + readonly passkeyBridgeArgument: string | undefined; +}): BrowserWindowConstructorOptions { + return { + width: BACKGROUND_TAB_WIDTH, + height: BACKGROUND_TAB_HEIGHT, + show: false, + paintWhenInitiallyHidden: true, + skipTaskbar: true, + webPreferences: { + partition: input.partition, + preload: input.preload, + // Mirrors `will-attach-webview` for panel guests. + sandbox: true, + nodeIntegration: false, + nodeIntegrationInSubFrames: false, + contextIsolation: false, + disableHtmlFullscreenWindowResize: true, + backgroundThrottling: false, + ...(input.passkeyBridgeArgument === undefined + ? {} + : { additionalArguments: [input.passkeyBridgeArgument] }), + }, + }; +} diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 04dcd22cf3b2c..762c7d3a3e2e2 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -55,10 +55,10 @@ describe("DesktopBrowserHost", () => { it.effect("announces tabs already attached to a backend that starts later", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; - host.attach(key, makeDebuggee().tab); + host.attach(key, makeDebuggee().tab, false); - expect(yield* takeEvents(host, 1)).toEqual([{ type: "attached", ...key }]); - expect(yield* takeEvents(host, 1)).toEqual([{ type: "attached", ...key }]); + expect(yield* takeEvents(host, 1)).toEqual([{ type: "attached", ...key, background: false }]); + expect(yield* takeEvents(host, 1)).toEqual([{ type: "attached", ...key, background: false }]); }), ); @@ -66,7 +66,7 @@ describe("DesktopBrowserHost", () => { Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; const debuggee = makeDebuggee(); - host.attach(key, debuggee.tab); + host.attach(key, debuggee.tab, false); const reader = yield* takeEvents(host, 2).pipe(Effect.forkScoped); yield* Effect.yieldNow; @@ -93,7 +93,7 @@ describe("DesktopBrowserHost", () => { Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; const debuggee = makeDebuggee(); - host.attach(key, debuggee.tab); + host.attach(key, debuggee.tab, false); const paths: Array = []; const item = { setSavePath: (path: string) => void paths.push(path), @@ -121,7 +121,7 @@ describe("DesktopBrowserHost", () => { Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; const debuggee = makeDebuggee(); - host.attach(key, debuggee.tab); + host.attach(key, debuggee.tab, false); yield* host.handleCommandLine( encodeJson({ diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index cef42d4f35fc3..2c9f6f1d8dbbd 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -42,6 +42,7 @@ const keyOf = ({ threadId, tabId }: DesktopBrowserTabKey) => `${threadId}\u0000$ interface AttachedTab { readonly key: DesktopBrowserTabKey; readonly debuggee: DesktopBrowserTabDebugger; + readonly background: boolean; relay: CdpRelayConnection | null; downloadDirectory: string | null; pendingDownloadGuid: string | null; @@ -59,7 +60,12 @@ export class DesktopBrowserHost extends Context.Service< { readonly events: Stream.Stream; readonly handleCommandLine: (line: string) => Effect.Effect; - readonly attach: (key: DesktopBrowserTabKey, debuggee: DesktopBrowserTabDebugger) => void; + /** `background` tabs live in a never-shown window that no panel sizes. */ + readonly attach: ( + key: DesktopBrowserTabKey, + debuggee: DesktopBrowserTabDebugger, + background: boolean, + ) => void; readonly detach: (key: DesktopBrowserTabKey) => void; readonly humanStartedDownload: (source: Electron.WebContents) => boolean; readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean; @@ -137,13 +143,18 @@ export const make = Effect.gen(function* () { emit({ type: "detached", ...key }); }; - const attach = (key: DesktopBrowserTabKey, debuggee: DesktopBrowserTabDebugger) => { + const attach = ( + key: DesktopBrowserTabKey, + debuggee: DesktopBrowserTabDebugger, + background: boolean, + ) => { const id = keyOf(key); if (tabs.get(id)?.debuggee.webContents === debuggee.webContents) return; detach(key); const tab: AttachedTab = { key, debuggee, + background, relay: null, downloadDirectory: null, pendingDownloadGuid: null, @@ -158,7 +169,7 @@ export const make = Effect.gen(function* () { }; tabs.set(id, tab); debuggee.debugger.on("message", tab.onMessage); - emit({ type: "attached", ...key }); + emit({ type: "attached", ...key, background }); }; const handleCommandLine = (line: string) => @@ -190,7 +201,7 @@ export const make = Effect.gen(function* () { [...tabs.values()], (tab) => { tab.relay = null; - return PubSub.publish(outbox, { type: "attached", ...tab.key }); + return PubSub.publish(outbox, { type: "attached", ...tab.key, background: tab.background }); }, { discard: true }, ), diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 6ded53bfaf1ba..6b67b010a57c9 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -600,6 +600,58 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "hosts an agent tab in a never-shown window that paints and closes with the tab", + () => + withManager((manager) => + Effect.gen(function* () { + const tab = makeTestPreviewWebContents( + async () => ({ + toJPEG: () => Buffer.from(""), + getSize: () => ({ width: 1, height: 1 }), + }), + 55, + null as unknown as TestHostWebContents, + ); + const loadURL = vi.fn(async () => undefined); + Object.assign(tab, { getType: () => "window", once: vi.fn(), loadURL, reload: vi.fn() }); + const window = { + webContents: tab, + once: vi.fn(), + isDestroyed: () => false, + destroy: vi.fn(), + }; + let options: Electron.BrowserWindowConstructorOptions | undefined; + browserWindowConstructor.mockImplementation(function ( + next: Electron.BrowserWindowConstructorOptions, + ) { + options = next; + return window; + }); + fromId.mockReturnValue(tab); + + const state = yield* manager.createTab( + "tab_agent", + { serverTab: { threadId: "thread", tabId: "agent" } }, + { partition: "persist:t3code-preview-test", url: "https://example.com/agent" }, + ); + + expect(options).toMatchObject({ + show: false, + paintWhenInitiallyHidden: true, + webPreferences: { + partition: "persist:t3code-preview-test", + backgroundThrottling: false, + }, + }); + expect(state.webContentsId).toBe(55); + expect(loadURL).toHaveBeenCalledWith("https://example.com/agent"); + yield* manager.closeTab("tab_agent"); + expect(window.destroy).toHaveBeenCalledOnce(); + }), + ), + ); + effectIt.effect("keeps editing and forwarded shortcuts in whichever tab holds focus", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index e9578c711ae40..70ce945061320 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -66,6 +66,8 @@ import * as DesktopRendererHistory from "../telemetry/DesktopRendererHistory.ts" import { MENU_ACTION_CHANNEL, PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL } from "../ipc/channels.ts"; import * as DesktopBrowserHost from "./DesktopBrowserHost.ts"; import * as PreviewPasskeys from "./Passkeys.ts"; +import { PASSKEY_BRIDGE_ARGUMENT } from "./GuestProtocol.ts"; +import { backgroundTabWindowOptions } from "./BackgroundTabWindow.ts"; import * as BrowserSession from "./BrowserSession.ts"; import { ANNOTATION_CAPTURED_CHANNEL, @@ -114,6 +116,12 @@ export interface PreviewTabState { updatedAt: string; } +/** An agent tab's never-shown window: the main-process-derived partition it browses with. */ +export interface PreviewBackgroundTab { + readonly partition: string; + readonly url?: string | undefined; +} + const ZOOM_LEVELS: ReadonlyArray = [ 0.25, 0.33, 0.5, 0.67, 0.75, 0.8, 0.9, 1.0, 1.1, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 4.0, 5.0, ]; @@ -549,6 +557,7 @@ const isPreviewInputSignal = (value: unknown): value is PreviewInputSignal => { const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function* ( artifactDirectory: string, pictureInPicturePreloadPath: string, + guestPreloadPath: string, ) { const fileSystem = yield* FileSystem.FileSystem; const rendererHistory = yield* DesktopRendererHistory.DesktopRendererHistory; @@ -1676,6 +1685,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (currentMainWindow !== window) return; currentMainWindow = undefined; frameCaptureWindowOpen = false; + // Off macOS the app quits once its last window closes, which a + // background tab would otherwise hold open. + if (hostPlatform !== "darwin") { + for (const tabId of backgroundWindows.keys()) closeBackgroundWindow(tabId); + } mainWindowCleanupFiber = runFork( Effect.all([closeAllPictureInPicture(), stopAllRecordings()], { concurrency: "unbounded", @@ -1733,11 +1747,67 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return result.state; }); + /** Agent tabs' never-shown windows; see backgroundTabWindowOptions. */ + const backgroundWindows = new Map(); + + const openBackgroundWindow = Effect.fn("PreviewManager.openBackgroundWindow")(function* ( + tabId: string, + background: PreviewBackgroundTab, + ) { + const window = yield* attempt( + { operation: "backgroundWindow.create", tabId }, + () => + new BrowserWindow( + backgroundTabWindowOptions({ + partition: background.partition, + preload: guestPreloadPath, + passkeyBridgeArgument: passkeys.bridgeEnabled ? PASSKEY_BRIDGE_ARGUMENT : undefined, + }), + ), + ); + backgroundWindows.set(tabId, window); + const wc = window.webContents; + window.once("closed", () => { + if (backgroundWindows.get(tabId) === window) backgroundWindows.delete(tabId); + }); + // A panel guest is remounted by the renderer after a crash; nothing else + // would bring this one back. + wc.on("render-process-gone", () => { + if (!wc.isDestroyed()) wc.reload(); + }); + yield* prepareWebview(wc); + yield* registerWebviewUnlocked(tabId, wc.id, tabLifecycleGenerations.get(tabId)); + if (background.url !== undefined) { + runFork( + attemptPromise({ operation: "backgroundWindow.load", tabId, webContentsId: wc.id }, () => + wc.loadURL(background.url!), + ).pipe(Effect.ignore), + ); + } + }); + + const closeBackgroundWindow = (tabId: string) => { + const window = backgroundWindows.get(tabId); + backgroundWindows.delete(tabId); + if (window && !window.isDestroyed()) window.destroy(); + }; + const createTab = Effect.fn("PreviewManager.createTab")(function* ( tabId: string, defaults?: DesktopPreviewTabDefaults, + background?: PreviewBackgroundTab, ) { - return yield* withTabLifecycleLock(tabId, createTabUnlocked(tabId, defaults)); + return yield* withTabLifecycleLock( + tabId, + Effect.gen(function* () { + const state = yield* createTabUnlocked(tabId, defaults); + if (background === undefined || backgroundWindows.has(tabId)) return state; + yield* openBackgroundWindow(tabId, background).pipe( + Effect.onError(() => Effect.sync(() => closeBackgroundWindow(tabId))), + ); + return (yield* SynchronizedRef.get(tabsRef)).get(tabId) ?? state; + }), + ); }); const closeTabUnlocked = Effect.fn("PreviewManager.closeTabUnlocked")(function* (tabId: string) { @@ -1776,6 +1846,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { concurrency: 2, discard: true }, ); } + closeBackgroundWindow(tabId); const updatedAt = yield* currentIso; const closed: PreviewTabState = { ...closedTab, @@ -1827,11 +1898,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } const wc = webContents.fromId(webContentsId); const mainWindow = yield* Ref.get(mainWindowRef); + const ownBackgroundWindow = backgroundWindows.get(tabId)?.webContents === wc; if ( !wc || wc.isDestroyed() || - wc.getType() !== "webview" || - (Option.isSome(mainWindow) && wc.hostWebContents !== mainWindow.value.webContents) + (!ownBackgroundWindow && + (wc.getType() !== "webview" || + (Option.isSome(mainWindow) && wc.hostWebContents !== mainWindow.value.webContents))) ) { return yield* new PreviewWebContentsNotFoundError({ tabId, webContentsId }); } @@ -2387,7 +2460,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } if (afterAttach.serverTab) { yield* listenForAgentPointers; - browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger }); + browserHost.attach( + afterAttach.serverTab, + { webContents: wc, debugger: control.debugger }, + backgroundWindows.get(tabId)?.webContents === wc, + ); } if (afterAttach.colorScheme !== "system") { yield* attemptPromise({ operation: "applyColorScheme", tabId, webContentsId: wc.id }, () => @@ -3595,6 +3672,7 @@ export class PreviewManager extends Context.Service< readonly createTab: ( tabId: string, defaults?: DesktopPreviewTabDefaults, + background?: PreviewBackgroundTab, ) => Effect.Effect; readonly closeTab: (tabId: string) => Effect.Effect; readonly registerWebview: ( @@ -3730,6 +3808,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { const operations = yield* makeNativeOperations( environment.browserArtifactsDir, environment.path.join(environment.dirname, "preview-pip-preload.cjs"), + environment.path.join(environment.dirname, "preview-pick-preload.cjs"), ); return PreviewManager.of({ diff --git a/apps/server/src/preview/DesktopBrowserChannel.ts b/apps/server/src/preview/DesktopBrowserChannel.ts index 0d1db856c1e6b..999ac681cbbb8 100644 --- a/apps/server/src/preview/DesktopBrowserChannel.ts +++ b/apps/server/src/preview/DesktopBrowserChannel.ts @@ -52,6 +52,8 @@ export class DesktopBrowserChannel extends Context.Service< readonly detached: Stream.Stream; readonly attached: Stream.Stream; readonly isAttached: (key: DesktopTabKey) => Effect.Effect; + /** The tab lives in a never-shown desktop window, so no panel sizes it. */ + readonly isBackground: (key: DesktopTabKey) => Effect.Effect; readonly endpoint: (key: DesktopTabKey) => Effect.Effect; readonly pointer: ( key: DesktopTabKey, @@ -65,7 +67,8 @@ const make = Effect.gen(function* () { const inputFd = config.desktopBrowserFd; const controlFd = config.desktopBrowserControlFd; const changes = yield* PubSub.unbounded<{ key: DesktopTabKey; attached: boolean }>(); - const attachedTabs = new Set(); + /** Attached tab ids, mapped to whether each is hosted in the background. */ + const attachedTabs = new Map(); const inbound = new Map>(); const writeLock = yield* Semaphore.make(1); @@ -76,6 +79,7 @@ const make = Effect.gen(function* () { detached: Stream.empty, attached: Stream.empty, isAttached: () => Effect.succeed(false), + isBackground: () => Effect.succeed(false), endpoint: () => Effect.die("No desktop app is attached to this server."), pointer: () => Effect.void, }); @@ -115,7 +119,7 @@ const make = Effect.gen(function* () { return queue ? Queue.offer(queue, event.message).pipe(Effect.asVoid) : Effect.void; } case "attached": - attachedTabs.add(id); + attachedTabs.set(id, event.background === true); return PubSub.publish(changes, { key, attached: true }); case "detached": { attachedTabs.delete(id); @@ -214,6 +218,7 @@ const make = Effect.gen(function* () { Stream.map((change) => change.key), ), isAttached: (key) => Effect.sync(() => attachedTabs.has(keyOf(key))), + isBackground: (key) => Effect.sync(() => attachedTabs.get(keyOf(key)) === true), endpoint, pointer: (key, pointer) => command({ type: "pointer", ...key, ...pointer }), }); diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index ac4a785f207f3..29b326bd3e252 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -221,6 +221,7 @@ let focusedFrame: ReturnType | null = null; let contextFailure: Error | null = null; let initialNavigation: (() => Promise) | null = null; let desktopRendersNext = false; +let desktopBackground = false; const desktopDetaches = new NodeEvents.EventEmitter(); const desktopTabs = new Set(); const desktopRenders = (tabId: string) => { @@ -286,6 +287,7 @@ const dependencies = Layer.mergeAll( ), ), isAttached: (key) => Effect.sync(() => desktopRenders(key.tabId)), + isBackground: () => Effect.sync(() => desktopBackground), endpoint: (key) => Effect.acquireRelease(Effect.succeed(`ws://desktop/${key.tabId}`), () => Effect.sync(() => releasedDesktopTabs.push(key.tabId)), @@ -329,6 +331,7 @@ beforeEach(() => { focusedFrame = null; desktopTabs.clear(); desktopRendersNext = false; + desktopBackground = false; releasedDesktopTabs.length = 0; desktopConnections.length = 0; }); @@ -494,6 +497,28 @@ it.live("enforces provider ownership and explicit targets when a session has mul ).pipe(Effect.provide(layer)), ); +it.live.each([ + { background: true, sized: true }, + { background: false, sized: false }, +])( + "a desktop tab is sized by the server only when no panel webview sizes it (background $background)", + ({ background, sized }) => + Effect.scoped( + Effect.gen(function* () { + desktopRendersNext = true; + desktopBackground = background; + const { browser, tabId } = yield* ready; + const page = desktopConnections[0]!.context.page; + page.setViewportSize.mockClear(); + const operator = yield* browser.attachViewer(viewerInput(tabId, true)); + yield* operator.input({ type: "takeControl" }); + yield* operator.input({ type: "resize", width: 390, height: 844 }); + if (sized) expect(page.setViewportSize).toHaveBeenCalledWith({ width: 390, height: 844 }); + else expect(page.setViewportSize).not.toHaveBeenCalled(); + }), + ).pipe(Effect.provide(layer)), +); + it.live("streams to a read-only viewer without allowing takeover, input, or viewport changes", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 5b386e5f0797a..53cb50ddad9c0 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -291,7 +291,12 @@ interface ServerTab { readonly isolatedContext: boolean; - readonly desktop: { readonly close: () => Promise } | null; + /** + * A desktop-rendered page. `background` pages live in a never-shown window, + * so the server sizes them like headless pages; the others follow the + * panel's ``. + */ + readonly desktop: { readonly close: () => Promise; readonly background: boolean } | null; readonly profileId: string | undefined; readonly openerTabId: string | undefined; @@ -505,10 +510,12 @@ const make = Effect.gen(function* () { } : null; + const inPanel = (tab: ServerTab) => tab.desktop !== null && !tab.desktop.background; + const updateFileChooserInterception = (tab: ServerTab) => { // Native desktop views use the OS picker. Only agents and streamed controllers // have an uploader that can answer an intercepted request. - const intercept = !tab.desktop || tab.agentOperating || tab.control.controller !== null; + const intercept = !inPanel(tab) || tab.agentOperating || tab.control.controller !== null; if (intercept === tab.fileChooserIntercepted) return; tab.fileChooserIntercepted = intercept; if (intercept) tab.page.on("filechooser", tab.onFileChooser); @@ -599,7 +606,7 @@ const make = Effect.gen(function* () { const layoutSize = (tab: ServerTab) => { const size = tab.page.viewportSize(); - if (!size || tab.desktop) return size; + if (!size || inPanel(tab)) return size; return { width: Math.max(1, Math.round(size.width / tab.zoomFactor)), height: Math.max(1, Math.round(size.height / tab.zoomFactor)), @@ -607,7 +614,7 @@ const make = Effect.gen(function* () { }; const applyZoom = async (tab: ServerTab) => { - if (tab.desktop) return; + if (inPanel(tab)) return; const size = tab.page.viewportSize(); if (!size) return; if (tab.zoomFactor === 1) { @@ -632,7 +639,7 @@ const make = Effect.gen(function* () { const applySetting = async (tab: ServerTab, setting: PreviewViewportSetting) => { tab.setting = setting; - if (tab.desktop) return; + if (inPanel(tab)) return; const size = fixedViewportSize(setting) ?? [...tab.viewers] @@ -652,7 +659,7 @@ const make = Effect.gen(function* () { await tab.page.emulateMedia({ colorScheme: colorScheme === "system" ? null : colorScheme }); } const zoomFactor = snapshot.zoomFactor ?? 1; - if (zoomFactor !== tab.zoomFactor && !tab.desktop) { + if (zoomFactor !== tab.zoomFactor && !inPanel(tab)) { tab.zoomFactor = zoomFactor; await applyZoom(tab); broadcastViewport(tab); @@ -701,8 +708,12 @@ const make = Effect.gen(function* () { .pipe(Scope.provide(scope)), ); const connected = await contexts.connectDesktopPage(endpoint); + const background = await Effect.runPromise( + desktopChannel.isBackground({ threadId: snapshot.threadId, tabId: snapshot.tabId }), + ); return { page: connected.page, + background, close: async () => { await connected.browser.close().catch(constVoid); await Effect.runPromise(Scope.close(scope, Exit.void)); @@ -764,7 +775,7 @@ const make = Effect.gen(function* () { fileChooserIntercepted: false, onFileChooser: (chooser) => void offerFileChooser(tab, chooser), isolatedContext, - desktop: desktop === null ? null : { close: desktop.close }, + desktop: desktop === null ? null : { close: desktop.close, background: desktop.background }, profileId: snapshot.profileId, openerTabId: adopted?.openerTabId, downloads: [], @@ -783,7 +794,7 @@ const make = Effect.gen(function* () { captureLock: Promise.resolve(), capturing: 0, }; - if (!desktop) { + if (!inPanel(tab)) { await page.setViewportSize(fixedViewportSize(tab.setting) ?? UNATTACHED_FILL_VIEWPORT); } await applyRendering(tab, snapshot); @@ -1273,14 +1284,14 @@ const make = Effect.gen(function* () { ); }; - // Desktop browsing bypasses streamed viewers and server input tracking, - // so these idle signals only apply to headless tabs. + // Panel browsing bypasses streamed viewers and server input tracking, + // so these idle signals only apply to tabs the panel does not render. const closeIdleAgentTabs = () => { const cutoff = Date.now() - AGENT_TAB_IDLE_MS; for (const tab of tabs.values()) { if ( tab.control.agentId !== null && - tab.desktop === null && + !inPanel(tab) && tab.viewers.size === 0 && tab.usedAt < cutoff ) @@ -2198,7 +2209,7 @@ const make = Effect.gen(function* () { const height = Math.min(Math.round(num(message.height)), 2160); if (width < 100 || height < 100) return; viewer.requestedSize = { width, height, order: ++viewerResizeOrder }; - if (tab.setting._tag !== "fill" || tab.desktop) return; + if (tab.setting._tag !== "fill" || inPanel(tab)) return; const current = tab.page.viewportSize(); if (current?.width === width && current.height === height) return; await tab.page.setViewportSize({ width, height }); diff --git a/apps/web/src/browser/ElectronBrowserHost.browser.test.tsx b/apps/web/src/browser/ElectronBrowserHost.browser.test.tsx index 29c217e064adc..f5aa7e06b8389 100644 --- a/apps/web/src/browser/ElectronBrowserHost.browser.test.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.browser.test.tsx @@ -22,6 +22,12 @@ const mocks = vi.hoisted(() => ({ vi.mock(import("~/env"), async (original) => ({ ...(await original()), isElectron: true })); +// Server tabs render on the desktop only for its own environment. +vi.mock(import("~/state/primaryEnvironment"), async () => { + const { Atom } = await import("effect/reactivity"); + return { primaryEnvironmentIdAtom: Atom.make(EnvironmentId.make("host-webview-order")) }; +}); + vi.mock(import("~/state/session"), async (original) => ({ ...(await original()), useEnvironmentScope: () => true, @@ -160,4 +166,34 @@ describe("ElectronBrowserHost", () => { expect(after).toHaveLength(2); expect(after.every((node, index) => node.isSameNode(before[index]!))).toBe(true); }); + + it("hosts an agent's tab in a background window instead of a panel webview", async () => { + using _state = setup(); + reconcilePreviewServerSessions(threadRef, { + sessions: [ + { + ...snapshot("tab_agent", "2026-01-01T00:00:01.000Z"), + runtime: "server", + automationOwner: "agent-session", + }, + ], + serverEpoch: "server-a", + revision: 1, + }); + await using screen = await renderHost(); + + await vi.waitFor(() => + expect(mocks.createTab).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + serverTab: { threadId: threadRef.threadId, tabId: "tab_agent" }, + background: expect.objectContaining({ + environmentId: threadRef.environmentId, + url: "https://example.com/tab_agent", + }), + }), + ), + ); + expect(screen.container.querySelectorAll("webview")).toHaveLength(0); + }); }); diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index 8818b377f5e3b..8ee85a341e239 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -3,6 +3,7 @@ import { AuthPreviewOperateScope, type DesktopPreviewOpenLinkEvent, FILL_PREVIEW_VIEWPORT, + type ScopedThreadRef, } from "@t3tools/contracts"; import { useAtomValue } from "@effect/atom-react"; import { type ComponentProps, useEffect, useEffectEvent, useState } from "react"; @@ -22,7 +23,8 @@ import { useBrowserDefaults } from "./browserDefaults"; import { useBrowserPointerStore } from "./browserPointerStore"; import { HostedBrowserWebview } from "./HostedBrowserWebview"; import { openUrlInPreview } from "./openFileInPreview"; -import { rendersServerTabNatively } from "./previewRuntime"; +import { acquireDesktopTab } from "./desktopTabLifetime"; +import { hostsServerTabInBackground, rendersServerTabNatively } from "./previewRuntime"; import { previewRuntimeTabId } from "./previewRuntimeTabId"; export function ElectronBrowserHost() { @@ -36,11 +38,17 @@ export function ElectronBrowserHost() { .filter( (snapshot) => snapshot.runtime !== "server" || - rendersServerTabNatively(threadRef.environmentId, primaryEnvironmentId, snapshot), + rendersServerTabNatively(threadRef.environmentId, primaryEnvironmentId, snapshot) || + hostsServerTabInBackground(threadRef.environmentId, primaryEnvironmentId, snapshot), ) .map((snapshot) => ({ threadRef, snapshot, + background: hostsServerTabInBackground( + threadRef.environmentId, + primaryEnvironmentId, + snapshot, + ), runtimeTabId: previewRuntimeTabId(threadRef, previewState.serverEpoch, snapshot.tabId), pictureInPicture: previewState.desktopByTabId[snapshot.tabId]?.pictureInPicture ?? false, @@ -116,35 +124,89 @@ export function ElectronBrowserHost() { if (!isElectron) return null; return (
- {sessions.map(({ threadRef, snapshot, runtimeTabId, pictureInPicture, zoomFactor }) => { - const url = snapshot.navStatus._tag === "Idle" ? null : snapshot.navStatus.url; - return ( - - ); - })} + {sessions.map( + ({ threadRef, snapshot, runtimeTabId, pictureInPicture, zoomFactor, background }) => { + const url = snapshot.navStatus._tag === "Idle" ? null : snapshot.navStatus.url; + if (background) { + return ( + + ); + } + return ( + + ); + }, + )}
); } +/** Holds the desktop tab open in its own window; the panel streams it from the server. */ +function BackgroundBrowserTab(props: { + readonly threadRef: ScopedThreadRef; + readonly tabId: string; + readonly runtimeTabId: string; + readonly initialUrl: string | null; + readonly profileId: string | undefined; +}) { + const { tabId, runtimeTabId } = props; + const { environmentId, threadId } = props.threadRef; + const canOperatePreview = useEnvironmentScope(environmentId, AuthPreviewOperateScope); + const profileId = useTabProfileId(props.profileId); + const hydrated = useClientSettingsHydrated(); + // Fixed at creation, like a webview's partition and first URL. + const [initialUrl] = useState(props.initialUrl); + useEffect(() => { + if (!canOperatePreview || !hydrated) return; + const lease = acquireDesktopTab( + runtimeTabId, + { threadId, tabId }, + { + environmentId, + ...(profileId === undefined ? {} : { profileId }), + ...(initialUrl === null ? {} : { url: initialUrl }), + }, + ); + return lease.release; + }, [ + canOperatePreview, + hydrated, + environmentId, + threadId, + tabId, + runtimeTabId, + profileId, + initialUrl, + ]); + return null; +} + function AuthorizedBrowserWebview(props: ComponentProps) { const canOperatePreview = useEnvironmentScope( props.threadRef.environmentId, diff --git a/apps/web/src/browser/desktopTabLifetime.ts b/apps/web/src/browser/desktopTabLifetime.ts index 5b5ed29eaa883..cdc95da9360f9 100644 --- a/apps/web/src/browser/desktopTabLifetime.ts +++ b/apps/web/src/browser/desktopTabLifetime.ts @@ -1,3 +1,5 @@ +import type { DesktopPreviewTabDefaults } from "@t3tools/contracts"; + import { previewBridge } from "~/components/preview/previewBridge"; import { browserDefaultTabState, resolveBrowserDefaults } from "./browserDefaults"; @@ -47,6 +49,7 @@ export interface AcquiredDesktopTab { export function acquireDesktopTab( tabId: string, serverTab?: { readonly threadId: string; readonly tabId: string }, + background?: DesktopPreviewTabDefaults["background"], ): AcquiredDesktopTab { const current = leases.get(tabId) ?? @@ -59,6 +62,7 @@ export function acquireDesktopTab( previewBridge?.createTab(tabId, { ...browserDefaultTabState(await resolveBrowserDefaults()), ...(serverTab === undefined ? {} : { serverTab }), + ...(background === undefined ? {} : { background }), }), ), } satisfies DesktopTabLease); diff --git a/apps/web/src/browser/previewRuntime.ts b/apps/web/src/browser/previewRuntime.ts index 3d8e1d0a1f43d..e4804ccfe2240 100644 --- a/apps/web/src/browser/previewRuntime.ts +++ b/apps/web/src/browser/previewRuntime.ts @@ -41,10 +41,13 @@ export function usePreviewAvailable(environmentId: EnvironmentId | null): boolea return isPreviewSupportedInRuntime() || serverBrowser; } -export function rendersServerTabNatively( +type RuntimeSnapshot = Pick; + +/** The desktop app hosts this server tab, in a panel `` or a background window. */ +function desktopHostsServerTab( environmentId: EnvironmentId, primaryEnvironmentId: EnvironmentId | null, - snapshot: Pick | null | undefined, + snapshot: RuntimeSnapshot | null | undefined, ): boolean { return ( isElectron && @@ -54,9 +57,36 @@ export function rendersServerTabNatively( ); } +export function rendersServerTabNatively( + environmentId: EnvironmentId, + primaryEnvironmentId: EnvironmentId | null, + snapshot: RuntimeSnapshot | null | undefined, +): boolean { + return ( + desktopHostsServerTab(environmentId, primaryEnvironmentId, snapshot) && + snapshot?.automationOwner === undefined + ); +} + +/** + * Agent tabs run in a never-shown desktop window, because a panel `` + * created while the app is hidden never paints and agents open tabs whenever + * they like. The panel streams them like any other server tab. + */ +export function hostsServerTabInBackground( + environmentId: EnvironmentId, + primaryEnvironmentId: EnvironmentId | null, + snapshot: RuntimeSnapshot | null | undefined, +): boolean { + return ( + desktopHostsServerTab(environmentId, primaryEnvironmentId, snapshot) && + snapshot?.automationOwner !== undefined + ); +} + export function useRendersServerTabNatively( environmentId: EnvironmentId, - snapshot: Pick | null | undefined, + snapshot: RuntimeSnapshot | null | undefined, ): boolean { return rendersServerTabNatively(environmentId, useAtomValue(primaryEnvironmentIdAtom), snapshot); } diff --git a/packages/contracts/src/desktopBrowser.ts b/packages/contracts/src/desktopBrowser.ts index c70b374910079..4e16d1f00712d 100644 --- a/packages/contracts/src/desktopBrowser.ts +++ b/packages/contracts/src/desktopBrowser.ts @@ -8,7 +8,15 @@ const TabKey = { }; export const DesktopBrowserEvent = Schema.Union([ - Schema.Struct({ type: Schema.Literal("attached"), ...TabKey }), + Schema.Struct({ + type: Schema.Literal("attached"), + ...TabKey, + /** + * Hosted in a never-shown window rather than a panel ``, so no + * panel sizes it and the server must. + */ + background: Schema.optional(Schema.Boolean), + }), /** Its `` went away: closed, crashed, swapped, or devtools took the debugger. */ Schema.Struct({ type: Schema.Literal("detached"), ...TabKey }), Schema.Struct({ type: Schema.Literal("cdp"), ...TabKey, message: Schema.String }), diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index d1f6b349397ee..307e08416c0e2 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -955,12 +955,30 @@ export const DesktopPreviewCreateTabInputSchema = Schema.Struct({ serverTab: Schema.optional( Schema.Struct({ threadId: TrimmedNonEmptyString, tabId: TrimmedNonEmptyString }), ), + /** + * Host the tab in a never-shown window instead of a panel ``. Main + * derives the partition from these, as it does for webview config. + */ + background: Schema.optional( + Schema.Struct({ + environmentId: EnvironmentId, + profileId: Schema.optional(BrowserProfileId), + url: Schema.optional(Schema.String), + }), + ), }); export interface DesktopPreviewTabDefaults { readonly zoomFactor?: number | undefined; readonly colorScheme?: DesktopPreviewColorScheme | undefined; readonly serverTab?: { readonly threadId: string; readonly tabId: string } | undefined; + readonly background?: + | { + readonly environmentId: EnvironmentId; + readonly profileId?: string | undefined; + readonly url?: string | undefined; + } + | undefined; } export const DesktopPreviewRegisterWebviewInputSchema = Schema.Struct({ diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 0e9ff94d6ffe5..12506dee42342 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -949,8 +949,10 @@ export class PreviewAutomationTimeoutError extends Schema.TaggedError