diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 99df18c10cf5a..2f09e2b084613 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -24,6 +24,7 @@ import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import type { BrowserContext, Page } from "playwright-core"; import { beforeEach, expect, vi } from "vite-plus/test"; @@ -56,6 +57,7 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ } async connectDesktopPage(endpoint: string) { const context = makeContext(); + context.page.viewportSize.mockReturnValue(null); desktopConnections.push({ endpoint, context }); return { browser: { close: async () => {} }, page: context.page as unknown as Page }; } @@ -83,11 +85,16 @@ const PNG_HEADER = (() => { return header.toString("base64"); })(); -function makeSession(frame?: ReturnType) { +function makeSession( + frame: ReturnType | undefined, + viewport: { width: number; height: number }, +) { const events = new NodeEvents.EventEmitter(); return { - on: vi.fn((name: string, listener: () => void) => events.on(name, listener)), - emit: (name: string) => events.emit(name), + on: vi.fn((name: string, callback: (...args: unknown[]) => void) => events.on(name, callback)), + emit: (name: string, ...args: unknown[]) => events.emit(name, ...args), + emitAsync: (name: string, ...args: unknown[]) => + Promise.all(events.listeners(name).map((listener) => listener(...args))), detach: vi.fn(async () => {}), send: vi.fn(async (method: string, input?: unknown): Promise> => { if (method === "Page.getNavigationHistory") return { currentIndex: 0, entries: [{}] }; @@ -101,7 +108,12 @@ function makeSession(frame?: ReturnType) { if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } }; if (method === "Page.createIsolatedWorld") { return { - executionContextId: (input as { worldName?: string }).worldName === "t3-preview" ? 1 : 7, + executionContextId: + (input as { worldName?: string }).worldName === "t3-preview-viewport" + ? 9 + : (input as { worldName?: string }).worldName === "t3-preview" + ? 1 + : 7, }; } if (method === "Runtime.evaluate" && (input as { contextId?: number }).contextId === 1) { @@ -118,6 +130,8 @@ function makeSession(frame?: ReturnType) { else pageCopy = ""; return { result: { value } }; } + if (method === "Runtime.evaluate" && (input as { contextId?: number }).contextId === 9) + return desktopSizeAnswer?.() ?? { result: { value: { ...viewport } } }; return { result: { value: "evaluated" } }; }), }; @@ -129,7 +143,8 @@ function makeContext(onClose?: (context: BrowserContext) => void) { const setChooserFiles = vi.fn(async (_files: unknown) => {}); const sessions: ReturnType[] = []; let url = "about:blank"; - let viewport = { width: 1280, height: 800 }; + let viewport: ReturnType = { width: 1280, height: 800 }; + const cssViewport = { width: 1920, height: 1080 }; let closed = false; let contextClosed = false; const page = { @@ -159,7 +174,7 @@ function makeContext(onClose?: (context: BrowserContext) => void) { frames: () => [page], url: () => url, title: vi.fn(async () => "test page"), - viewportSize: () => viewport, + viewportSize: vi.fn(() => viewport), setViewportSize: vi.fn(async (size: typeof viewport) => { viewport = size; }), @@ -197,11 +212,13 @@ function makeContext(onClose?: (context: BrowserContext) => void) { const context = { page, sessions, + cssViewport, newPage: async () => page as unknown as Page, grantPermissions: vi.fn(async () => {}), newCDPSession: async (target: unknown) => { const session = makeSession( target === page ? undefined : (target as ReturnType), + cssViewport, ); sessions.push(session); return session; @@ -268,7 +285,11 @@ let focusedFrame: ReturnType | null = null; let contextFailure: Error | null = null; let initialNavigation: (() => Promise) | null = null; let desktopRendersNext = false; +/** Replaces the answer to the desktop page's size reads. */ +let desktopSizeAnswer: (() => Promise>) | null = null; const desktopDetaches = new NodeEvents.EventEmitter(); +let desktopDetachReady = Promise.withResolvers(); +let desktopReleased = Promise.withResolvers(); const desktopTabs = new Set(); const desktopRenders = (tabId: string) => { if (desktopRendersNext) { @@ -317,6 +338,7 @@ const dependencies = Layer.mergeAll( const onDetach = (key: { threadId: string; tabId: string }) => Queue.offerUnsafe(queue, key); desktopDetaches.on("detach", onDetach); + desktopDetachReady.resolve(); return onDetach; }), (onDetach) => Effect.sync(() => desktopDetaches.off("detach", onDetach)), @@ -336,7 +358,10 @@ const dependencies = Layer.mergeAll( isAttached: (key) => Effect.sync(() => desktopRenders(key.tabId)), endpoint: (key) => Effect.acquireRelease(Effect.succeed(`ws://desktop/${key.tabId}`), () => - Effect.sync(() => releasedDesktopTabs.push(key.tabId)), + Effect.sync(() => { + releasedDesktopTabs.push(key.tabId); + desktopReleased.resolve(); + }), ), pointer: () => Effect.void, turn: (threadId, active) => Effect.sync(() => desktopTurns.emit("turn", threadId, active)), @@ -367,6 +392,33 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({ maxHeight: 800, quality: 70, }); +/** Fills a viewer's output with frames; resolves once every one of them is acknowledged. */ +const stallViewer = (viewer: ServerBrowser.ServerBrowserViewer) => { + const output = viewer.output as unknown as Queue.Queue; + const acked = Promise.withResolvers(); + let unacked = 0; + const ack = Effect.sync(() => { + if (--unacked === 0) acked.resolve(); + }); + while (Queue.offerUnsafe(output, { _tag: "frame", data: Buffer.alloc(0), ack })) unacked++; + return Effect.promise(() => acked.promise); +}; +const viewportSizes = (items: ReadonlyArray) => + items.flatMap((item) => (item._tag === "viewport" ? [`${item.width}x${item.height}`] : [])); +const openDesktopViewer = Effect.gen(function* () { + const browser = yield* ServerBrowser.ServerBrowser; + const manager = yield* Manager.PreviewManager; + desktopRendersNext = true; + const { tabId } = yield* manager.open({ threadId: scope.thread.threadId, runtime: "server" }); + const viewer = yield* browser.attachViewer(viewerInput(tabId, true)); + const context = desktopConnections[0]!.context; + const cdp = context.sessions[0]!; + const resize = (width: number, height: number) => { + Object.assign(context.cssViewport, { width, height }); + return Effect.promise(() => cdp.emitAsync("Page.frameResized")); + }; + return { tabId, viewer, context, cdp, resize }; +}); beforeEach(() => { contexts.length = 0; @@ -380,6 +432,9 @@ beforeEach(() => { focusedFrame = null; desktopTabs.clear(); desktopRendersNext = false; + desktopSizeAnswer = null; + desktopDetachReady = Promise.withResolvers(); + desktopReleased = Promise.withResolvers(); releasedDesktopTabs.length = 0; desktopConnections.length = 0; }); @@ -1144,6 +1199,148 @@ it.live("only the current controller can answer a file picker", () => ).pipe(Effect.provide(layer)), ); +it.effect( + "desktop viewers get the page's CSS size and keep the last one a read cannot measure", + () => + Effect.scoped( + Effect.gen(function* () { + // Each page size read waits here for its answer. + const reads = yield* Queue.unbounded>>(); + const deferReads = () => { + desktopSizeAnswer = () => { + const read = Promise.withResolvers>(); + Queue.offerUnsafe(reads, read); + return read.promise; + }; + }; + deferReads(); + const { viewer, cdp, context } = yield* openDesktopViewer; + const answer = (width: number, height: number) => ({ + result: { value: { width, height } }, + }); + const resized = () => cdp.emitAsync("Page.frameResized"); + // The read on attach never answers and gives up with no size to send. + const blocked = yield* Queue.take(reads); + yield* TestClock.adjust("1 second"); + // The next page load measures again. + desktopSizeAnswer = null; + yield* Effect.promise(() => cdp.emitAsync("Page.loadEventFired")); + expect(viewportSizes(yield* Queue.takeAll(viewer.output))).toEqual(["1920x1080"]); + // While a dialog is open nothing reads. The host closing it clears it, and since + // closing resizes nothing, the close itself remeasures. + deferReads(); + context.page.emit("dialog", { + type: () => "alert", + message: () => "", + defaultValue: () => "", + }); + yield* Effect.promise(resized); + expect(yield* Queue.size(reads)).toBe(0); + desktopSizeAnswer = null; + yield* Effect.promise(() => cdp.emitAsync("Page.javascriptDialogClosed")); + expect(viewportSizes(yield* Queue.takeAll(viewer.output))).toEqual([ + "1920x1080", + "1920x1080", + ]); + // The late answer, a read a navigation breaks and a hidden webview keep the last size. + blocked.resolve(answer(100, 100)); + deferReads(); + const navigated = resized(); + (yield* Queue.take(reads)).reject(new Error("Cannot find context with specified id")); + yield* Effect.promise(() => navigated); + const hidden = resized(); + (yield* Queue.take(reads)).resolve(answer(0, 0)); + yield* Effect.promise(() => hidden); + expect(viewportSizes(yield* Queue.takeAll(viewer.output))).toEqual([ + "1920x1080", + "1920x1080", + ]); + // Resizes during a read share one more read, so the newest size arrives last. + const first = resized(); + const firstRead = yield* Queue.take(reads); + const burst = Promise.all([resized(), resized()]); + firstRead.resolve(answer(800, 500)); + (yield* Queue.take(reads)).resolve(answer(1000, 600)); + expect(yield* Queue.size(reads)).toBe(0); + yield* Effect.promise(() => Promise.all([first, burst])); + expect(viewportSizes(yield* Queue.takeAll(viewer.output))).toEqual([ + "800x500", + "1000x600", + "1000x600", + ]); + }), + ).pipe(Effect.provide(layer)), +); + +it.effect("a stalled desktop viewer still gets its size, control and reconnect", () => + Effect.scoped( + Effect.gen(function* () { + const { tabId, viewer, context, resize } = yield* openDesktopViewer; + const output = + viewer.output as unknown as Queue.Queue; + // Ready once the sizes measured on attach and on a first resize both arrived. + yield* resize(1920, 1080); + expect(viewportSizes(yield* Queue.takeAll(viewer.output))).toEqual([ + "1920x1080", + "1920x1080", + ]); + yield* Effect.promise(() => desktopDetachReady.promise); + yield* viewer.input({ type: "takeControl" }); + context.page.emit("filechooser", { + isMultiple: () => false, + element: () => ({ getAttribute: async () => ".csv" }), + setFiles: vi.fn(async () => {}), + }); + let chooser = yield* Queue.take(viewer.output); + while (chooser._tag !== "fileChooser") chooser = yield* Queue.take(viewer.output); + yield* Queue.clear(viewer.output); + // A resize compacts a full backlog but keeps control, the open picker and + // one-time notifications in order. + yield* viewer.input({ type: "takeControl" }); + const notifications: Array = [ + { _tag: "fileChooserClosed", id: "chooser-0" }, + { _tag: "popup", tabId: "popup-1" }, + { _tag: "download", id: "download-1", fileName: "report.csv", sizeBytes: 3 }, + { _tag: "clipboard", text: "copied" }, + ]; + for (const item of notifications) Queue.offerUnsafe(output, item); + let acked = stallViewer(viewer); + yield* resize(1000, 600); + expect(yield* Queue.takeAll(viewer.output)).toEqual([ + chooser, + ...notifications, + expect.objectContaining({ _tag: "control", controller: "you" }), + { _tag: "viewport", width: 1000, height: 600 }, + ]); + yield* acked; + // A control change keeps only the newest size. + yield* resize(900, 500); + yield* resize(1200, 700); + acked = stallViewer(viewer); + yield* viewer.input({ type: "releaseControl" }); + expect(yield* Queue.takeAll(viewer.output)).toEqual([ + expect.objectContaining({ _tag: "control", controller: "unclaimed" }), + { _tag: "viewport", width: 1200, height: 700 }, + ]); + yield* acked; + // Notifications that no longer fit go oldest first, never the size. + let copies = 0; + while (Queue.offerUnsafe(output, { _tag: "clipboard", text: `copy ${copies}` })) copies++; + yield* resize(1000, 600); + const compacted = yield* Queue.takeAll(viewer.output); + expect(compacted.at(0)).toEqual({ _tag: "clipboard", text: "copy 1" }); + expect(compacted.at(-1)).toEqual({ _tag: "viewport", width: 1000, height: 600 }); + // The desktop takes the page back; a resize after it cannot bury the reconnect. + acked = stallViewer(viewer); + desktopDetaches.emit("detach", { threadId: scope.thread.threadId, tabId }); + yield* Effect.promise(() => desktopReleased.promise); + yield* resize(1000, 600); + expect(yield* Queue.takeAll(viewer.output)).toEqual([{ _tag: "reconnect" }]); + yield* acked; + }), + ).pipe(Effect.provide(layer)), +); + it.live("agents read a human's tab with no arguments and act on it only while nobody drives", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 6f3e86a7501ea..57e35be778634 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -102,6 +102,8 @@ const RECORDING_SCREENCAST = { format: "jpeg", quality: 90, everyNthFrame: 1 } a const decodeViewportSetting = Schema.decodeUnknownSync(PreviewViewportSettingSchema); const AGENT_CURSOR_MOVE_MS = 160; const AGENT_CURSOR_CLICK_LEAD_MS = 40; +/** A dialog blocks page script, so a desktop size read gives up after this. */ +const DESKTOP_VIEWPORT_READ_TIMEOUT = "1 second"; const sleepUntil = (deadline: number) => new Promise((resolve) => setTimeout(resolve, Math.max(0, deadline - Date.now()))); @@ -309,6 +311,11 @@ interface ServerTab { setting: PreviewViewportSetting; colorScheme: PreviewAppearancePreference; zoomFactor: number; + /** Last usable CSS size measured on the desktop. */ + desktopViewport: { readonly width: number; readonly height: number } | null; + desktopViewportRead: Promise; + /** A read queued behind `desktopViewportRead`, shared by every caller that waits for it. */ + desktopViewportNext: Promise | null; loading: boolean; closing: boolean; recording: Recording | null; @@ -605,9 +612,48 @@ const make = Effect.gen(function* () { ); }; - const layoutSize = (tab: ServerTab) => { + // Desktop pages have no Playwright viewport; an isolated world keeps page script + // from overriding the CSS size, and failed reads preserve the last usable size. + const readDesktopViewport = async (tab: ServerTab) => { + const readSize = async () => { + const { frameTree } = await tab.cdp.send("Page.getFrameTree"); + const { executionContextId } = await tab.cdp.send("Page.createIsolatedWorld", { + frameId: frameTree.frame.id, + worldName: "t3-preview-viewport", + }); + const { result } = await tab.cdp.send("Runtime.evaluate", { + expression: "({ width: innerWidth, height: innerHeight })", + contextId: executionContextId, + returnByValue: true, + }); + return asRecord(result.value); + }; + const size = await runPromise( + // A navigation can destroy the isolated world mid-read. + Effect.promise(() => readSize().catch(() => null)).pipe( + Effect.timeoutOption(DESKTOP_VIEWPORT_READ_TIMEOUT), + Effect.map(Option.getOrNull), + ), + ); + const width = size?.width; + const height = size?.height; + if (typeof width === "number" && width > 0 && typeof height === "number" && height > 0) + tab.desktopViewport = { width, height }; + return tab.desktopViewport; + }; + + // Reads run one at a time, and callers that arrive during a read share the next one. + const measureDesktopViewport = (tab: ServerTab) => + (tab.desktopViewportNext ??= tab.desktopViewportRead.then(() => { + tab.desktopViewportNext = null; + return (tab.desktopViewportRead = readDesktopViewport(tab)); + })); + + const layoutSize = async (tab: ServerTab) => { + // A dialog blocks page script; its close measures again. + if (tab.desktop) return tab.dialog ? tab.desktopViewport : measureDesktopViewport(tab); const size = tab.page.viewportSize(); - if (!size || tab.desktop) return size; + if (!size) return size; return { width: Math.max(1, Math.round(size.width / tab.zoomFactor)), height: Math.max(1, Math.round(size.height / tab.zoomFactor)), @@ -622,7 +668,7 @@ const make = Effect.gen(function* () { await tab.page.setViewportSize(size); return; } - const layout = layoutSize(tab)!; + const layout = (await layoutSize(tab))!; await tab.cdp.send("Emulation.setDeviceMetricsOverride", { width: layout.width, height: layout.height, @@ -631,9 +677,9 @@ const make = Effect.gen(function* () { }); }; - const broadcastViewport = (tab: ServerTab) => { - const size = layoutSize(tab); - if (!size) return; + const broadcastViewport = async (tab: ServerTab) => { + const size = await layoutSize(tab); + if (!size || tab.closing) return; for (const viewer of tab.viewers) viewer.push({ _tag: "viewport", ...size }); }; @@ -650,7 +696,7 @@ const make = Effect.gen(function* () { UNATTACHED_FILL_VIEWPORT; await tab.page.setViewportSize({ width: size.width, height: size.height }); await applyZoom(tab); - broadcastViewport(tab); + await broadcastViewport(tab); }; const applyRendering = async (tab: ServerTab, snapshot: PreviewSessionSnapshot) => { @@ -663,7 +709,7 @@ const make = Effect.gen(function* () { if (zoomFactor !== tab.zoomFactor && !tab.desktop) { tab.zoomFactor = zoomFactor; await applyZoom(tab); - broadcastViewport(tab); + await broadcastViewport(tab); } }; @@ -789,6 +835,9 @@ const make = Effect.gen(function* () { setting: snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, colorScheme: "system", zoomFactor: 1, + desktopViewport: null, + desktopViewportRead: Promise.resolve(null), + desktopViewportNext: null, loading: false, closing: false, recording: null, @@ -803,6 +852,21 @@ const make = Effect.gen(function* () { await page.setViewportSize(fixedViewportSize(tab.setting) ?? UNATTACHED_FILL_VIEWPORT); } await applyRendering(tab, snapshot); + if (desktop) { + // The desktop resizes and zooms its webview itself; viewers map input with the new size. + const remeasure = () => { + if (tab.viewers.size > 0) return broadcastViewport(tab).catch(constVoid); + }; + cdp.on("Page.frameResized", remeasure); + // A read a navigation broke has a fresh page to measure. + cdp.on("Page.loadEventFired", remeasure); + // Closing a dialog resizes nothing, and the host can close it without us. + cdp.on("Page.javascriptDialogClosed", () => { + tab.dialog = null; + return remeasure(); + }); + await cdp.send("Page.enable"); + } const isMainNavigation = (request: { isNavigationRequest(): boolean; frame(): unknown }) => request.isNavigationRequest() && request.frame() === page.mainFrame(); const navigationGenerations = new WeakMap(); @@ -1383,7 +1447,7 @@ const make = Effect.gen(function* () { }; } const url = tab.page.url(); - const viewport = tab.page.viewportSize(); + const viewport = tab.desktop ? await layoutSize(tab) : tab.page.viewportSize(); const status = { available: true, visible: tab.viewers.size > 0, @@ -2213,7 +2277,7 @@ const make = Effect.gen(function* () { const current = tab.page.viewportSize(); if (current?.width === width && current.height === height) return; await tab.page.setViewportSize({ width, height }); - broadcastViewport(tab); + await broadcastViewport(tab); return; } case "viewport": { @@ -2313,15 +2377,22 @@ const make = Effect.gen(function* () { // Dropped frames must still release Chromium. if (Queue.offerUnsafe(output, next)) return; if (next._tag === "frame") runFork(next.ack); - // State a stalled viewer cannot miss replaces its backlog. It runs - // synchronously so an older replacement can never land after a newer one. - else if (next._tag === "gone" || next._tag === "control" || next._tag === "fileChooser") { + // State a stalled viewer cannot miss compacts its backlog. It runs + // synchronously so an older compaction can never land after a newer one. + else if (["gone", "reconnect", "control", "viewport", "fileChooser"].includes(next._tag)) { const dropped = Effect.runSyncExit(Queue.clear(output)); if (dropped._tag === "Failure") return; - Queue.offerUnsafe(output, next); - - const chooser = next._tag === "control" ? fileChooserMessage(tab) : null; - if (chooser && tab.control.controller === viewer.id) Queue.offerUnsafe(output, chooser); + const items = [...dropped.value, next]; + // Without its latest size and control the viewer cannot map input at all. + const state = (["control", "viewport"] as const).flatMap( + (tag) => items.findLast((item) => item._tag === tag) ?? [], + ); + // Everything else is one-time; if it no longer fits, the oldest goes first. + const rest = items.filter( + (item) => item._tag !== "frame" && item._tag !== "control" && item._tag !== "viewport", + ); + for (const item of [...rest.slice(state.length - VIEWER_OUTPUT_LIMIT), ...state]) + Queue.offerUnsafe(output, item); for (const item of dropped.value) if (item._tag === "frame") runFork(item.ack); } }, @@ -2410,7 +2481,7 @@ const make = Effect.gen(function* () { ), }); }); - broadcastViewport(tab); + void broadcastViewport(tab).catch(constVoid); yield* Effect.promise(() => startScreencast(1)); // An idle page does not repaint for a new screencast, so the viewer // starts from a still unless a live frame beat it.