fix(desktop): snapshot background preview tabs without timing out

apps/desktop/src/preview/DesktopBrowserHost.test.ts

@@ -6,6 +6,7 @@ import * as Fiber from "effect/Fiber";
 import * as Queue from "effect/Queue";
 import * as Schema from "effect/Schema";
 import * as Stream from "effect/Stream";
+import * as TestClock from "effect/testing/TestClock";
 import * as NodeEvents from "node:events";
 
 import * as DesktopBrowserHost from "./DesktopBrowserHost.ts";
@@ -30,24 +31,37 @@ const command = (
     }),
   );
 
-const makeDebuggee = () => {
+const makeDebuggee = (
+  prepareSurface: () => Promise<unknown> = async () => true,
+  restoreSurface: () => Promise<unknown> = async () => true,
+) => {
   const emitter = new NodeEvents.EventEmitter();
-  const pending: Array<() => void> = [];
+  const pending: Array<(cause?: Error) => void> = [];
   const sent: Array<unknown> = [];
   const sendWaiters: Array<() => void> = [];
   const debuggee = Object.assign(emitter, {
     sendCommand: (method: string, params?: { id?: number }) =>
-      new Promise((resolve) => {
+      new Promise((resolve, reject) => {
         if (method === "Target.getTargetInfo") {
           resolve({ targetInfo: { targetId: "GUEST" } });
           return;
         }
         sent.push(params?.id);
-        pending.push(() => resolve({ method }));
+        pending.push((cause) => (cause ? reject(cause) : resolve({ method })));
         sendWaiters.shift()?.();
       }),
   });
+  const surfaceRequests: string[] = [];
   const webContents = Object.assign(new NodeEvents.EventEmitter(), {
+    id: 42,
+    isDestroyed: () => false,
+    hostWebContents: {
+      isDestroyed: () => false,
+      executeJavaScript: (expression: string) => {
+        surfaceRequests.push(expression);
+        return expression.endsWith(", true)") ? prepareSurface() : restoreSurface();
+      },
+    },
     getURL: () => "http://localhost/",
     getTitle: () => "Page",
     getUserAgent: () => "Electron",
@@ -61,9 +75,10 @@ const makeDebuggee = () => {
     navigate: () => webContents.emit("did-navigate"),
     /** Ids of the commands the debugger received, in order. */
     sent,
+    surfaceRequests,
     nextSend: () => new Promise<void>((resolve) => sendWaiters.push(resolve)),
     release: () => pending.splice(0).forEach((resolve) => resolve()),
-    releaseFirst: () => pending.shift()?.(),
+    releaseFirst: (cause?: Error) => pending.shift()?.(cause),
   };
 };
 
@@ -74,6 +89,16 @@ const takeEvents = (host: DesktopBrowserHost.DesktopBrowserHost["Service"], coun
     Effect.map((lines) => lines.map((line) => decodeEvent(new TextDecoder().decode(line)))),
   );
 
+const sendScreenshot = Effect.fn("test.sendScreenshot")(function* (
+  host: DesktopBrowserHost.DesktopBrowserHost["Service"],
+  debuggee: ReturnType<typeof makeDebuggee>,
+  id: number,
+) {
+  const sent = debuggee.nextSend();
+  yield* command(host, id, "Page.captureScreenshot");
+  yield* Effect.promise(() => sent);
+});
+
 /**
  * Ids of the replies the host sends, in order. A screenshot's reply leaves after its turn
  * has passed on, so by then the next waiting screenshot, if any, has been sent. Subscribe
@@ -94,6 +119,96 @@ const watchReplies = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) =>
   });
 
 describe("DesktopBrowserHost", () => {
+  it.effect.each([false, true])(
+    "waits for a renderable surface and restores it after capture (failed: %s)",
+    (failed) =>
+      Effect.gen(function* () {
+        const host = yield* DesktopBrowserHost.make;
+        const ready = Promise.withResolvers<boolean>();
+        const debuggee = makeDebuggee(() => ready.promise);
+        host.attach(key, debuggee.tab);
+        const replies = yield* watchReplies(host);
+
+        yield* command(host, 1, "Page.captureScreenshot");
+        expect(debuggee.sent).toEqual([]);
+
+        const sent = debuggee.nextSend();
+        ready.resolve(true);
+        yield* Effect.promise(() => sent);
+        debuggee.releaseFirst(failed ? new Error("Capture failed") : undefined);
+        expect(yield* Queue.take(replies)).toBe(1);
+        expect(debuggee.surfaceRequests).toHaveLength(2);
+      }).pipe(Effect.scoped),
+  );
+
+  it.effect("does not send a screenshot superseded while its surface was becoming ready", () =>
+    Effect.gen(function* () {
+      const host = yield* DesktopBrowserHost.make;
+      const ready = Promise.withResolvers<boolean>();
+      const debuggee = makeDebuggee(() => ready.promise);
+      host.attach(key, debuggee.tab);
+      const replies = yield* watchReplies(host);
+
+      yield* command(host, 1, "Page.captureScreenshot");
+      debuggee.navigate();
+      ready.resolve(true);
+      expect(yield* Queue.take(replies)).toBe(1);
+      expect(debuggee.sent).toEqual([]);
+      expect(debuggee.surfaceRequests).toHaveLength(2);
+    }).pipe(Effect.scoped),
+  );
+
+  it.effect.each<[string, () => Promise<unknown>]>([
+    ["never becomes ready", async () => false],
+    ["can't be reached", () => Promise.reject(new Error("Script failed"))],
+  ])("still issues a screenshot when its render surface %s", ([, prepareSurface]) =>
+    Effect.gen(function* () {
+      const host = yield* DesktopBrowserHost.make;
+      const debuggee = makeDebuggee(prepareSurface);
+      host.attach(key, debuggee.tab);
+      const replies = yield* watchReplies(host);
+
+      yield* sendScreenshot(host, debuggee, 1);
+      debuggee.releaseFirst();
+      expect(yield* Queue.take(replies)).toBe(1);
+      expect(debuggee.sent).toEqual([1]);
+      expect(debuggee.surfaceRequests).toHaveLength(2);
+    }).pipe(Effect.scoped),
+  );
+
+  it.effect("sends the screenshot when the renderer never answers readiness", () =>
+    Effect.gen(function* () {
+      const host = yield* DesktopBrowserHost.make;
+      const debuggee = makeDebuggee(() => new Promise(() => {}));
+      host.attach(key, debuggee.tab);
+      const replies = yield* watchReplies(host);
+
+      const sent = debuggee.nextSend();
+      yield* command(host, 1, "Page.captureScreenshot");
+      yield* TestClock.adjust("2 seconds");
+      yield* Effect.promise(() => sent);
+      debuggee.releaseFirst();
+      expect(yield* Queue.take(replies)).toBe(1);
+    }).pipe(Effect.scoped),
+  );
+
+  it.effect("answers and moves on when the renderer never answers restoring the surface", () =>
+    Effect.gen(function* () {
+      const host = yield* DesktopBrowserHost.make;
+      const debuggee = makeDebuggee(undefined, () => new Promise(() => {}));
+      host.attach(key, debuggee.tab);
+      const replies = yield* watchReplies(host);
+
+      yield* sendScreenshot(host, debuggee, 1);
+      yield* command(host, 2, "Page.captureScreenshot");
+      const secondSent = debuggee.nextSend();
+      debuggee.releaseFirst();
+      expect(yield* Queue.take(replies)).toBe(1);
+      yield* Effect.promise(() => secondSent);
+      expect(debuggee.sent).toEqual([1, 2]);
+    }).pipe(Effect.scoped),
+  );
+
   it.effect("announces tabs already attached to a backend that starts later", () =>
     Effect.gen(function* () {
       const host = yield* DesktopBrowserHost.make;
@@ -120,7 +235,7 @@ describe("DesktopBrowserHost", () => {
             message: encodeJson({ id, method, sessionId: "t3-preview-page" }),
           }),
         );
-      yield* command(1, "Page.captureScreenshot");
+      yield* sendScreenshot(host, debuggee, 1);
       yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
       yield* command(2, "DOM.enable");
       debuggee.release();
@@ -138,13 +253,15 @@ describe("DesktopBrowserHost", () => {
       host.attach(key, debuggee.tab);
       const replies = yield* watchReplies(host);
 
-      yield* command(host, 1, "Page.captureScreenshot");
+      yield* sendScreenshot(host, debuggee, 1);
       yield* command(host, 2, "Page.captureScreenshot");
       yield* command(host, 3, "DOM.enable");
       expect(debuggee.sent).toEqual([1, 3]);
 
+      const secondSent = debuggee.nextSend();
       debuggee.releaseFirst();
       expect(yield* Queue.take(replies)).toBe(1);
+      yield* Effect.promise(() => secondSent);
       expect(debuggee.sent).toEqual([1, 3, 2]);
     }).pipe(Effect.scoped),
   );
@@ -156,19 +273,23 @@ describe("DesktopBrowserHost", () => {
       host.attach(key, debuggee.tab);
       const replies = yield* watchReplies(host);
 
-      yield* command(host, 1, "Page.captureScreenshot");
+      yield* sendScreenshot(host, debuggee, 1);
       yield* command(host, 2, "Page.captureScreenshot");
       yield* command(host, 3, "Page.captureScreenshot");
+      const secondSent = debuggee.nextSend();
       debuggee.navigate();
+      yield* Effect.promise(() => secondSent);
       expect(debuggee.sent).toEqual([1, 2]);
 
       // The superseded screenshot answering late must not start a third beside the second.
       debuggee.releaseFirst();
       expect(yield* Queue.take(replies)).toBe(1);
       expect(debuggee.sent).toEqual([1, 2]);
 
+      const thirdSent = debuggee.nextSend();
       debuggee.releaseFirst();
       expect(yield* Queue.take(replies)).toBe(2);
+      yield* Effect.promise(() => thirdSent);
       expect(debuggee.sent).toEqual([1, 2, 3]);
     }).pipe(Effect.scoped),
   );
@@ -179,7 +300,7 @@ describe("DesktopBrowserHost", () => {
       const debuggee = makeDebuggee();
       host.attach(key, debuggee.tab);
 
-      yield* command(host, 1, "Page.captureScreenshot");
+      yield* sendScreenshot(host, debuggee, 1);
       yield* command(host, 2, "Page.captureScreenshot");
       yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
       yield* command(host, 3, "Page.captureScreenshot");

apps/desktop/src/preview/DesktopBrowserHost.ts

@@ -4,12 +4,14 @@
  * the relay shares it.
  */
 import {
+  DESKTOP_PREVIEW_CAPTURE_SURFACE_TRIGGER,
   DesktopBrowserCommand,
   DesktopBrowserEvent,
   type DesktopBrowserEvent as DesktopBrowserEventType,
 } from "@t3tools/contracts";
 import * as Context from "effect/Context";
 import * as NodePath from "node:path";
+import * as NodeCrypto from "node:crypto";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
@@ -26,6 +28,8 @@ const lineEncoder = new TextEncoder();
 const AGENT_INPUT_COMMAND =
   /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/;
 const AGENT_DOWNLOAD_WINDOW_MS = 5_000;
+// The renderer gives up on readiness after 1s; this only catches a renderer that never answers.
+const CAPTURE_SURFACE_READY_TIMEOUT = "2 seconds";
 
 export interface DesktopBrowserTabKey {
   readonly threadId: string;
@@ -84,6 +88,7 @@ export const make = Effect.gen(function* () {
     readonly y: number;
   }>(16);
   const runFork = Effect.runForkWith(yield* Effect.context<never>());
+  const runPromise = Effect.runPromiseWith(yield* Effect.context<never>());
   const tabs = new Map<string, AttachedTab>();
   const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event));
 
@@ -112,7 +117,7 @@ export const make = Effect.gen(function* () {
         }
         const turn = {};
         tab.screenshotTurn = turn;
-        capture()
+        withCaptureSurface(tab, relay, turn, capture)
           .finally(() => {
             if (tab.screenshotTurn === turn) nextScreenshot(tab);
           })
@@ -122,6 +127,42 @@ export const make = Effect.gen(function* () {
       else tab.waitingScreenshots.push(start);
     });
 
+  const withCaptureSurface = async (
+    tab: AttachedTab,
+    relay: CdpRelayConnection,
+    turn: object,
+    capture: () => Promise<unknown>,
+  ) => {
+    const wc = tab.debuggee.webContents;
+    const host = wc.hostWebContents;
+    if (!host || host.isDestroyed() || wc.isDestroyed()) return capture();
+    const captureId = NodeCrypto.randomUUID();
+    const setSurface = (active: boolean) =>
+      host.executeJavaScript(
+        `globalThis[${JSON.stringify(DESKTOP_PREVIEW_CAPTURE_SURFACE_TRIGGER)}]?.(${wc.id}, ${JSON.stringify(captureId)}, ${active})`,
+      );
+    try {
+      // The guest stays offscreen until React commits the activity lease and Electron
+      // receives its new bounds. CDP can otherwise wait forever for a composited frame.
+      // A window that isn't painting, like a minimized one, never reports ready, yet its
+      // on-screen tab can still be captured, so the screenshot is sent either way.
+      // A renderer that never answers would hold this screenshot and every one queued behind it.
+      await runPromise(
+        Effect.tryPromise(() => setSurface(true)).pipe(
+          Effect.timeoutOption(CAPTURE_SURFACE_READY_TIMEOUT),
+          Effect.ignore,
+        ),
+      );
+      if (tab.relay !== relay || tab.screenshotTurn !== turn || wc.isDestroyed()) {
+        throw new Error("The preview connection that asked for this screenshot closed.");
+      }
+      return await capture();
+    } finally {
+      // The renderer bounds its own lease, so the result needn't wait for its answer.
+      if (!host.isDestroyed()) void setSurface(false).catch(() => undefined);
+    }
+  };
+
   const relayFor = (tab: AttachedTab) => {
     if (tab.relay) return tab.relay;
     const { webContents, debugger: debuggee } = tab.debuggee;

apps/web/src/browser/ElectronBrowserHost.tsx

@@ -19,13 +19,15 @@ import { useAtomCommand } from "~/state/use-atom-command";
 
 import { readPreviewAnnotationTheme } from "./annotationTheme";
 import { useBrowserDefaults } from "./browserDefaults";
+import { installBrowserCaptureSurface } from "./browserCaptureSurface";
 import { useBrowserPointerStore } from "./browserPointerStore";
 import { HostedBrowserWebview } from "./HostedBrowserWebview";
 import { openUrlInPreview } from "./openFileInPreview";
 import { rendersServerTabNatively } from "./previewRuntime";
 import { previewRuntimeTabId } from "./previewRuntimeTabId";
 
 export function ElectronBrowserHost() {
+  useEffect(() => (isElectron ? installBrowserCaptureSurface() : undefined), []);
   const previewByThreadKey = useActivePreviewSessions();
   const primaryEnvironmentId = useAtomValue(primaryEnvironmentIdAtom);
   useDesktopHostedPreviewSessions(primaryEnvironmentId);